From bc870bb95812a9e56a09b710562987dd7f256210 Mon Sep 17 00:00:00 2001 From: Ludmila Marian Date: Wed, 4 Nov 2015 12:05:54 +0100 Subject: [PATCH 01/59] WebSearch: fix the order of prev and next links * FIX Fixes the order of search results when computing the next and previous search links on the record page by not reversing the initial search results. Signed-off-by: Ludmila Marian --- modules/websearch/lib/websearch_templates.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/websearch/lib/websearch_templates.py b/modules/websearch/lib/websearch_templates.py index 2c54ea5596..f2a36e2135 100644 --- a/modules/websearch/lib/websearch_templates.py +++ b/modules/websearch/lib/websearch_templates.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # This file is part of Invenio. -# Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 CERN. +# Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 CERN. # # Invenio is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -4022,7 +4022,7 @@ def tmpl_display_back_to_search(self, req, recID, ln): # let's calculate lenght of recID's collection if record_found: - recIDs = coll_recID[::-1] + recIDs = coll_recID totalrec = len(recIDs) # search for a specific record having not done any search before else: From da162943abca92c1dfa3f29a90bdfb03a316531e Mon Sep 17 00:00:00 2001 From: David Zerulla Date: Tue, 3 May 2016 17:22:36 +0200 Subject: [PATCH 02/59] BibFormat: optional record recommender * NEW Record recommender loads and displays recommendations on each record page. The recommendations are calculated with the speared python package the invenio-record-recommender. * Loads the calculated recommendations from Redis. * Creates /record/*id*/recommendations endpoint. * Introduces CFG_RECOMMENDER_REDIS and CFG_RECOMMENDER_PREFIX config used to set the Redis instance from where the recommendations should be loaded. * Renders recommended records in JavaScript after the record is loaded. * Logs clicks on recommended records to webstat. Signed-off-by: David Zerulla --- config/invenio.conf | 15 ++ .../Default_HTML_detailed.bft | 2 + modules/bibformat/lib/elements/Makefile.am | 1 + .../elements/bfe_record_recommendations.py | 115 ++++++++++++++ modules/miscutil/lib/Makefile.am | 4 +- modules/miscutil/lib/recommender.py | 149 ++++++++++++++++++ .../miscutil/lib/recommender_initializer.py | 45 ++++++ .../websearch/lib/websearch_webinterface.py | 51 +++++- modules/webstat/etc/webstat.cfg | 8 + modules/webstyle/css/invenio.css | 29 +++- 10 files changed, 415 insertions(+), 4 deletions(-) create mode 100644 modules/bibformat/lib/elements/bfe_record_recommendations.py create mode 100644 modules/miscutil/lib/recommender.py create mode 100644 modules/miscutil/lib/recommender_initializer.py diff --git a/config/invenio.conf b/config/invenio.conf index 408411ba6a..27f895f3bd 100644 --- a/config/invenio.conf +++ b/config/invenio.conf @@ -2565,6 +2565,21 @@ CFG_ARXIV_URL_PATTERN = http://export.arxiv.org/pdf/%sv%s.pdf # e.g. CFG_REDIS_HOSTS = [{'db': 0, 'host': '127.0.0.1', 'port': 7001}] CFG_REDIS_HOSTS = {'default': [{'db': 0, 'host': '127.0.0.1', 'port': 6379}]} + +############################## +# Recommender Configuration ## +############################## +# CFG_RECOMMENDER_REDIS -- optionally, enables the recommendations and +# specifies the Redis host from where the recommendations are loaded. +# To show the recommendations on the record page include the +# BibFormat element `bfe_record_recommendations`. +CFG_RECOMMENDER_REDIS = + +# CFG_RECOMMENDER_PREFIX -- optionally, defines the prefix used in the +# redis cache. +CFG_RECOMMENDER_PREFIX = Reco_1:: + + ########################## # THAT's ALL, FOLKS! ## ########################## diff --git a/modules/bibformat/etc/format_templates/Default_HTML_detailed.bft b/modules/bibformat/etc/format_templates/Default_HTML_detailed.bft index 314a598a4a..96987994e5 100644 --- a/modules/bibformat/etc/format_templates/Default_HTML_detailed.bft +++ b/modules/bibformat/etc/format_templates/Default_HTML_detailed.bft @@ -45,4 +45,6 @@ suffix="
" /> + + diff --git a/modules/bibformat/lib/elements/Makefile.am b/modules/bibformat/lib/elements/Makefile.am index f628e635c4..24b1395d52 100644 --- a/modules/bibformat/lib/elements/Makefile.am +++ b/modules/bibformat/lib/elements/Makefile.am @@ -83,6 +83,7 @@ pylib_DATA = __init__.py \ bfe_record_id.py \ bfe_record_stats.py \ bfe_record_url.py \ + bfe_record_recommendations.py \ bfe_references.py \ bfe_report_numbers.py \ bfe_reprints.py \ diff --git a/modules/bibformat/lib/elements/bfe_record_recommendations.py b/modules/bibformat/lib/elements/bfe_record_recommendations.py new file mode 100644 index 0000000000..b7cc5a8d69 --- /dev/null +++ b/modules/bibformat/lib/elements/bfe_record_recommendations.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# +# This file is part of Invenio. +# Copyright (C) 2015, 2016 CERN. +# +# Invenio is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# Invenio is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Invenio; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +"""BibFormat element. + +* Creates a list of record recommendations +""" + +import re + +from invenio.config import CFG_RECOMMENDER_REDIS +from invenio.messages import gettext_set_language + + +html_script = """ + +""" + + +def format_element(bfo): + """Create the HTML and JS code to display the recommended records.""" + if CFG_RECOMMENDER_REDIS == "": + return "" + try: + re.search(r'/record/', bfo.user_info.get('uri')).group() + except AttributeError: + # No record url found + return "" + + _ = gettext_set_language(bfo.lang) + + url = "/record/" + str(bfo.recID) + "/recommendations" + html = html_script % {'recommendations_url': url, + 'text_title': _("You might also be interested in"), + } + return html + + +def escape_values(bfo): + """Called by BibFormat to check if output should be escaped.""" + return 0 diff --git a/modules/miscutil/lib/Makefile.am b/modules/miscutil/lib/Makefile.am index 2d094d2711..8281cc1053 100644 --- a/modules/miscutil/lib/Makefile.am +++ b/modules/miscutil/lib/Makefile.am @@ -102,7 +102,9 @@ pylib_DATA = __init__.py \ hepdatautils_unit_tests.py \ filedownloadutils.py \ filedownloadutils_unit_tests.py \ - viafutils.py + viafutils.py \ + recommender_initializer.py \ + recommender.py jsdir=$(localstatedir)/www/js diff --git a/modules/miscutil/lib/recommender.py b/modules/miscutil/lib/recommender.py new file mode 100644 index 0000000000..6df6cbfec3 --- /dev/null +++ b/modules/miscutil/lib/recommender.py @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- +# +# This file is part of Invenio. +# Copyright (C) 2016 CERN. +# +# Invenio is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# Invenio is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Invenio; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +"""Loads recommendations and optionally formats and filters them.""" + +import json + +from invenio.bibfield import get_record +from invenio.config import CFG_BASE_URL, \ + CFG_RECOMMENDER_PREFIX, \ + CFG_SITE_RECORD +from invenio.recommender_initializer import get_redis_connection +from invenio.webuser import collect_user_info + + +def get_recommended_records(recid): + """ + Record recommendations. + + @param recid: Record id for the recommendation. + @return: List of recommended records and a version ([234, 34, 2], 5) + """ + redis_connector = get_redis_connection() + + if not redis_connector: + return [], 0 + + # Get recommendation + key = "{0}{1}".format(CFG_RECOMMENDER_PREFIX, recid) + recommendations = redis_connector.get(key) + if not recommendations: + # No recommendations available. + return [], 0 + + recommendations = json.loads(recommendations) + + records = recommendations.get('records', []) + recommender_version = recommendations.get('version', 0) + + return records, recommender_version + + +def get_recommended_records_with_metadata(recid, maximum=3): + """ + Record recommendations with metadata as title, authors and url. + + @param recid: Record id for the recommendation. + @param maximum: Maximum recommendations to return. + @return: List of recommended records [{ + 'number': , + 'record_url': , + 'record_title': , + 'record_authors': , + }, ] + """ + records, recommender_version = get_recommended_records(recid) + return _format_recommendation(records, recid, recommender_version, + user_id=0, maximum=maximum) + + +def _format_recommendation(recommended_records, recid_source, + recommender_version, user_id=0, + maximum=3): + """ + Load the record information for each record in a list. + + @param recommended_records: List of records [1234 ,344 ,342] + @param recid_source: Record id where the recommendations are displayed. + @param recommender_version: Recommender version (for custom events). + @param user_id: User ID checks if the user has access to the recommended + records, user id 0 shows only public records. + @param maximum: Maximum recommendations to return. + @return: List of recommended records [{ + 'number': , + 'record_url': , + 'record_title': , + 'record_authors': , + }, ] + """ + from invenio.webstat import get_url_customevent + from invenio.search_engine import check_user_can_view_record, \ + record_public_p + + check_same_title = [] + suggestions = [] + rec_count = 1 + for recid in recommended_records: + try: + if user_id > 0: + # check if user can view record + user_info = collect_user_info(user_id) + if check_user_can_view_record(user_info, recid)[0] > 0: + # Not authorized + continue + else: + # check if record is public + if not record_public_p(recid): + continue + + # get record information + record = get_record(recid) + title = record.get('title.title') + + # Check for no title or similar title + if not title or title in check_same_title: + continue + else: + check_same_title.append(title) + + rec_authors = filter(None, record.get('authors.full_name', [])) \ + or filter(None, record.get('corporate_name.name', [])) + authors = "; ".join(rec_authors) + except (KeyError, TypeError, ValueError, AttributeError): + continue + + record_url = "{0}/{1}/{2}".format(CFG_BASE_URL, CFG_SITE_RECORD, + str(recid)) + url = get_url_customevent(record_url, + "recommended_record", + [str(recid_source), str(recid), + str(rec_count), str(user_id), + str(recommender_version)]) + suggestions.append({ + 'number': rec_count, + 'record_url': url, + 'record_title': title.strip(), + 'record_authors': authors.strip(), + }) + if rec_count >= maximum: + break + rec_count += 1 + + return suggestions diff --git a/modules/miscutil/lib/recommender_initializer.py b/modules/miscutil/lib/recommender_initializer.py new file mode 100644 index 0000000000..c0e7f119a4 --- /dev/null +++ b/modules/miscutil/lib/recommender_initializer.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# +# This file is part of Invenio. +# Copyright (C) 2016 CERN. +# +# Invenio is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# Invenio is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Invenio; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +"""Initializes the Redis connection.""" + +from invenio.config import CFG_RECOMMENDER_REDIS + + +_RECOMMENDATION_REDIS = None + + +def get_redis_connection(): + """ + Stores a persistent Redis connection. + + @return: Redis connection object. + """ + global _RECOMMENDATION_REDIS + + if CFG_RECOMMENDER_REDIS == "": + # Recommender is not configured. + return None + + if _RECOMMENDATION_REDIS is None: + import redis + _RECOMMENDATION_REDIS = redis.StrictRedis(host=CFG_RECOMMENDER_REDIS, + port=6379, + db=0) + return _RECOMMENDATION_REDIS diff --git a/modules/websearch/lib/websearch_webinterface.py b/modules/websearch/lib/websearch_webinterface.py index d016f66ede..68d3a1db3b 100644 --- a/modules/websearch/lib/websearch_webinterface.py +++ b/modules/websearch/lib/websearch_webinterface.py @@ -1,5 +1,5 @@ # This file is part of Invenio. -# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 CERN. +# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2016 CERN. # # Invenio is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -108,6 +108,7 @@ from invenio.bibdocfile_webinterface import WebInterfaceManageDocFilesPages, WebInterfaceFilesPages from invenio.bibfield import get_record from invenio.shellutils import mymkdir +from invenio.recommender import get_recommended_records_with_metadata import invenio.template websearch_templates = invenio.template.load('websearch') @@ -200,7 +201,8 @@ class WebInterfaceRecordPages(WebInterfaceDirectory): _exports = ['', 'files', 'reviews', 'comments', 'usage', 'references', 'export', 'citations', 'holdings', 'edit', 'keywords', - 'multiedit', 'merge', 'plots', 'linkbacks', 'hepdata'] + 'multiedit', 'merge', 'plots', 'linkbacks', 'hepdata', + 'recommendations'] #_exports.extend(output_formats) @@ -223,6 +225,7 @@ def __init__(self, recid, tab, form=None): self.edit = WebInterfaceEditPages(self.recid) self.merge = WebInterfaceMergePages(self.recid) self.linkbacks = WebInterfaceRecordLinkbacksPages(self.recid) + self.recommendations = WebInterfaceRecordRecommendations(self.recid) return @@ -303,6 +306,50 @@ def __call__(self, req, form): # Return the same page wether we ask for /CFG_SITE_RECORD/123 or /CFG_SITE_RECORD/123/ index = __call__ + +class WebInterfaceRecordRecommendations(WebInterfaceDirectory): + """Handling of a /CFG_SITE_RECORD//recommendations URL fragment.""" + + _exports = [''] + + def __init__(self, recid): + self.recid = recid + + def __call__(self, req, form): + """Called in case of URLs like /CFG_SITE_RECORD/123/recommendations.""" + req.content_type = 'application/json' + + user_info = collect_user_info(req) + uid = user_info['uid'] + + if uid == -1 or CFG_ACCESS_CONTROL_LEVEL_SITE > 1: + return {'status': 'error', "code": 1, "message": "Not authorized"} + + (auth_code, auth_message) = check_user_can_view_record(user_info, + self.recid) + if auth_code > 0: + # Not authorized + return {'status': 'error', "code": 1, "message": "Not authorized"} + + result = {} + result['items'] = get_recommended_records_with_metadata(self.recid, + maximum=3) + result['loggedin'] = True if uid > 0 else False + result['status'] = 'ok' + + try: + output = json.dumps(result) + except UnicodeDecodeError as e: + e.level = logging.WARN + register_exception(alert_admin=True, prefix="UnicodeDecodeError, " + "Record needs to be fixed") + result = {'status': 'error', "code": 2, + "message": "UnicodeDecodeError"} + output = json.dumps(result) + + return output + + class WebInterfaceRecordRestrictedPages(WebInterfaceDirectory): """ Handling of a /record-restricted/ URL fragment """ diff --git a/modules/webstat/etc/webstat.cfg b/modules/webstat/etc/webstat.cfg index ed28f4bc9c..bcb5bd1d7e 100644 --- a/modules/webstat/etc/webstat.cfg +++ b/modules/webstat/etc/webstat.cfg @@ -57,6 +57,14 @@ param2 = key_id param3 = path param4 = query +[webstat_custom_event_8] +name = recommended_record +param1 = recid_from +param2 = recid_to +param3 = rec_pos +param4 = user_id +param5 = recommender + [apache_log_analyzer] profile = nil nb-histogram-items-to-print = 20 diff --git a/modules/webstyle/css/invenio.css b/modules/webstyle/css/invenio.css index 288e75eaa2..f0ca40abb3 100644 --- a/modules/webstyle/css/invenio.css +++ b/modules/webstyle/css/invenio.css @@ -2,7 +2,7 @@ * -*- mode: text; coding: utf-8; -*- This file is part of Invenio. -Copyright (C) 2009, 2010, 2011, 2012, 2013, 2015 CERN. +Copyright (C) 2009, 2010, 2011, 2012, 2013, 2015, 2016 CERN. Invenio is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as @@ -4048,4 +4048,31 @@ a.author_orcid_image_link { margin-right:10px; } +/* Record Recommendation */ +.recommendations { + padding: 10px 10px; + box-sizing: border-box; + border: 1px solid #ccc; + width: 90%; + max-width: 1280px; + margin: 40px auto 0px; +} + +.recommendations_contents { + width: 50%; + display: inline; +} + +.recommendation_header { + display: block; + border-bottom: 1px solid #ccc; + padding-bottom: 10px; +} + +ul.record_recommendation li{ + clear: both; + color: #444; + margin-top:5px; +} + /* end of invenio.css */ From bc66faac0d8c0639feb8ef99f0f33f841a33a0b7 Mon Sep 17 00:00:00 2001 From: Patrick Glauner Date: Mon, 23 Sep 2013 20:31:32 +0200 Subject: [PATCH 03/59] SolrUtils: reliable regression test suite * Makes test suite independent from PDF extraction process. (closes #1590) * Simplifies test suite. Signed-off-by: Patrick Glauner --- .../miscutil/lib/solrutils_bibrank_indexer.py | 3 + .../lib/solrutils_regression_tests.py | 292 ++++++------------ 2 files changed, 95 insertions(+), 200 deletions(-) diff --git a/modules/miscutil/lib/solrutils_bibrank_indexer.py b/modules/miscutil/lib/solrutils_bibrank_indexer.py index fd0f83da19..e54e41e7c2 100644 --- a/modules/miscutil/lib/solrutils_bibrank_indexer.py +++ b/modules/miscutil/lib/solrutils_bibrank_indexer.py @@ -35,6 +35,9 @@ from invenio.bibrank_bridge_utils import get_tags, get_field_content_in_utf8 +SOLR_CONNECTION = None + + if CFG_SOLR_URL: import solr SOLR_CONNECTION = solr.SolrConnection(CFG_SOLR_URL) # pylint: disable=E1101 diff --git a/modules/miscutil/lib/solrutils_regression_tests.py b/modules/miscutil/lib/solrutils_regression_tests.py index 5dddd58d3e..fe471985f4 100644 --- a/modules/miscutil/lib/solrutils_regression_tests.py +++ b/modules/miscutil/lib/solrutils_regression_tests.py @@ -24,30 +24,50 @@ from invenio import intbitset from invenio.solrutils_bibindex_searcher import solr_get_bitset from invenio.solrutils_bibrank_searcher import solr_get_ranked, solr_get_similar_ranked +from invenio.solrutils_bibrank_indexer import solr_add, SOLR_CONNECTION from invenio.search_engine import get_collection_reclist from invenio.bibrank_bridge_utils import get_external_word_similarity_ranker, \ get_logical_fields, \ get_tags, \ get_field_content_in_utf8 + ROWS = 100 HITSETS = { 'Willnotfind': intbitset.intbitset([]), - 'higgs': intbitset.intbitset([47, 48, 51, 52, 55, 56, 58, 68, 79, 85, 89, 96]), - 'of': intbitset.intbitset([8, 10, 11, 12, 15, 43, 44, 45, 46, 47, 48, 49, 50, 51, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 64, 68, 74, - 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, - 91, 92, 93, 94, 95, 96, 97]), - '"higgs boson"': intbitset.intbitset([55, 56]), + 'of': intbitset.intbitset([1, 2, 7, 8, 10]), + '"of the"': intbitset.intbitset([1, 2, 7, 8, 10]) } -def get_topN(n, data): - res = dict() - for key, value in data.iteritems(): - res[key] = value[-n:] - return res + +RECORDS = xrange(1, 11) + + +TAGS = {'abstract': ['520__%'], + 'author': ['100__a', '700__a'], + 'keyword': ['6531_a'], + 'title': ['245__%', '246__%']} + + +def init_Solr(): + _delete_all() + _index_records() + SOLR_CONNECTION.commit() + + +def _delete_all(): + SOLR_CONNECTION.delete_query('*:*') + + +def _index_records(): + for recid in RECORDS: + fulltext = abstract = get_field_content_in_utf8(recid, 'abstract', TAGS) + author = get_field_content_in_utf8(recid, 'author', TAGS) + keyword = get_field_content_in_utf8(recid, 'keyword', TAGS) + title = get_field_content_in_utf8(recid, 'title', TAGS) + solr_add(recid, abstract, author, fulltext, keyword, title) class TestSolrSearch(InvenioTestCase): @@ -55,23 +75,19 @@ class TestSolrSearch(InvenioTestCase): make install-solrutils CFG_SOLR_URL set fulltext index in idxINDEX containing 'SOLR' in indexer column - AND EITHER - Solr index built: ./bibindex -w fulltext for all records - OR - WRD method referring to Solr: /etc/bibrank$ cp template_word_similarity_solr.cfg wrd.cfg - and ./bibrank -w wrd for all records """ - - def _get_result(self, query, index='fulltext'): + def get_result(self, query, index='fulltext'): return solr_get_bitset(index, query) + def setUp(self): + init_Solr() + @nottest def test_get_bitset(self): """solrutils - search results""" - self.assertEqual(HITSETS['Willnotfind'], self._get_result('Willnotfind')) - self.assertEqual(HITSETS['higgs'], self._get_result('higgs')) - self.assertEqual(HITSETS['of'], self._get_result('of')) - self.assertEqual(HITSETS['"higgs boson"'], self._get_result('"higgs boson"')) + self.assertEqual(self.get_result('Willnotfind'), HITSETS['Willnotfind']) + self.assertEqual(self.get_result('of'), HITSETS['of']) + self.assertEqual(self.get_result('"of the"'), HITSETS['"of the"']) class TestSolrRanking(InvenioTestCase): @@ -79,91 +95,25 @@ class TestSolrRanking(InvenioTestCase): make install-solrutils CFG_SOLR_URL set fulltext index in idxINDEX containing 'SOLR' in indexer column - AND EITHER - Solr index built: ./bibindex -w fulltext for all records - OR - WRD method referring to Solr: /etc/bibrank$ cp template_word_similarity_solr.cfg wrd.cfg - and ./bibrank -w wrd for all records """ - - def _get_ranked_result_sequence(self, query, index='fulltext', rows=ROWS, hitset=None): - if hitset is None: - hitset=HITSETS[query] - ranked_result = solr_get_ranked('%s:%s' % (index, query), hitset, self._get_ranking_params(), rows) + def get_ranked_result_sequence(self, query, index='fulltext', rows=ROWS): + ranked_result = solr_get_ranked('%s:%s' % (index, query), + HITSETS[query], + {'cutoff_amount': 10000, + 'cutoff_time_ms': 2000 + }, + rows) return tuple([pair[0] for pair in ranked_result[0]]) - def _get_ranked_topN(self, n): - return get_topN(n, self._RANKED) - - _RANKED = { - 'Willnotfind': tuple(), - 'higgs': (79, 51, 55, 47, 56, 96, 58, 68, 52, 48, 89, 85), - 'of': (50, 61, 60, 54, 56, 53, 10, 68, 44, 57, 83, 95, 92, 91, 74, 45, 48, 62, 82, - 49, 51, 89, 90, 96, 43, 8, 64, 97, 15, 85, 78, 46, 55, 79, 84, 88, 81, 52, - 58, 86, 11, 80, 93, 77, 12, 59, 87, 47, 94), - '"higgs boson"': (55, 56), - } - - def _get_ranking_params(self, cutoff_amount=10000, cutoff_time=2000): - """ - Default values from template_word_similarity_solr.cfg - """ - return { - 'cutoff_amount': cutoff_amount, - 'cutoff_time_ms': cutoff_time - } + def setUp(self): + init_Solr() @nottest def test_get_ranked(self): """solrutils - ranking results""" - all_ranked = 0 - ranked_top = self._get_ranked_topN(all_ranked) - self.assertEqual(ranked_top['Willnotfind'], self._get_ranked_result_sequence(query='Willnotfind')) - self.assertEqual(ranked_top['higgs'], self._get_ranked_result_sequence(query='higgs')) - self.assertEqual(ranked_top['of'], self._get_ranked_result_sequence(query='of')) - self.assertEqual(ranked_top['"higgs boson"'], self._get_ranked_result_sequence(query='"higgs boson"')) - - @nottest - def test_get_ranked_top(self): - """solrutils - ranking top results""" - top_n = 0 - self.assertEqual(tuple(), self._get_ranked_result_sequence(query='Willnotfind', rows=top_n)) - self.assertEqual(tuple(), self._get_ranked_result_sequence(query='higgs', rows=top_n)) - self.assertEqual(tuple(), self._get_ranked_result_sequence(query='of', rows=top_n)) - self.assertEqual(tuple(), self._get_ranked_result_sequence(query='"higgs boson"', rows=top_n)) - - top_n = 2 - ranked_top = self._get_ranked_topN(top_n) - self.assertEqual(ranked_top['Willnotfind'], self._get_ranked_result_sequence(query='Willnotfind', rows=top_n)) - self.assertEqual(ranked_top['higgs'], self._get_ranked_result_sequence(query='higgs', rows=top_n)) - self.assertEqual(ranked_top['of'], self._get_ranked_result_sequence(query='of', rows=top_n)) - self.assertEqual(ranked_top['"higgs boson"'], self._get_ranked_result_sequence(query='"higgs boson"', rows=top_n)) - - top_n = 10 - ranked_top = self._get_ranked_topN(top_n) - self.assertEqual(ranked_top['Willnotfind'], self._get_ranked_result_sequence(query='Willnotfind', rows=top_n)) - self.assertEqual(ranked_top['higgs'], self._get_ranked_result_sequence(query='higgs', rows=top_n)) - self.assertEqual(ranked_top['of'], self._get_ranked_result_sequence(query='of', rows=top_n)) - self.assertEqual(ranked_top['"higgs boson"'], self._get_ranked_result_sequence(query='"higgs boson"', rows=top_n)) - - @nottest - def test_get_ranked_smaller_hitset(self): - """solrutils - ranking smaller hitset""" - hitset = intbitset.intbitset([47, 56, 58, 68, 85, 89]) - self.assertEqual((47, 56, 58, 68, 89, 85), self._get_ranked_result_sequence(query='higgs', hitset=hitset)) - - hitset = intbitset.intbitset([45, 50, 61, 74, 94]) - self.assertEqual((50, 61, 74, 45, 94), self._get_ranked_result_sequence(query='of', hitset=hitset)) - self.assertEqual((74, 45, 94), self._get_ranked_result_sequence(query='of', hitset=hitset, rows=3)) - - @nottest - def test_get_ranked_larger_hitset(self): - """solrutils - ranking larger hitset""" - hitset = intbitset.intbitset([47, 56, 58, 68, 85, 89]) - self.assertEqual(tuple(), self._get_ranked_result_sequence(query='Willnotfind', hitset=hitset)) - - hitset = intbitset.intbitset([47, 56, 55, 56, 58, 68, 85, 89]) - self.assertEqual((55, 56), self._get_ranked_result_sequence(query='"higgs boson"', hitset=hitset)) + self.assertEqual(self.get_ranked_result_sequence(query='Willnotfind'), tuple()) + self.assertEqual(self.get_ranked_result_sequence(query='of'), (8, 2, 1, 10, 7)) + self.assertEqual(self.get_ranked_result_sequence(query='"of the"'), (8, 10, 1, 2, 7)) class TestSolrSimilarToRecid(InvenioTestCase): @@ -172,69 +122,37 @@ class TestSolrSimilarToRecid(InvenioTestCase): CFG_SOLR_URL set fulltext index in idxINDEX containing 'SOLR' in indexer column WRD method referring to Solr: /etc/bibrank$ cp template_word_similarity_solr.cfg wrd.cfg - ./bibrank -w wrd for all records """ - - def _get_similar_result_sequence(self, recid, rows=ROWS): - similar_result = solr_get_similar_ranked(recid, self._all_records, self._get_similar_ranking_params(), rows) + def get_similar_result_sequence(self, recid, rows=ROWS): + similar_result = solr_get_similar_ranked(recid, + self._all_records, + {'cutoff_amount': 10000, + 'cutoff_time_ms': 2000, + 'find_similar_to_recid': { + 'more_results_factor': 5, + 'mlt_fl': 'mlt', + 'mlt_mintf': 0, + 'mlt_mindf': 0, + 'mlt_minwl': 0, + 'mlt_maxwl': 0, + 'mlt_maxqt': 25, + 'mlt_maxntp': 1000, + 'mlt_boost': 'false' + } + }, + rows) return tuple([pair[0] for pair in similar_result[0]])[-rows:] - def _get_similar_topN(self, n): - return get_topN(n, self._SIMILAR) - - _SIMILAR = { - 30: (12, 95, 85, 82, 44, 1, 89, 64, 58, 15, 96, 61, 50, 86, 78, 77, 65, 62, 60, - 47, 46, 100, 99, 102, 91, 80, 7, 5, 92, 88, 74, 57, 55, 108, 84, 81, 79, 54, - 101, 11, 103, 94, 48, 83, 72, 63, 2, 68, 51, 53, 97, 93, 70, 45, 52, 14, - 59, 6, 10, 32, 33, 29, 30), - 59: (17, 69, 3, 20, 109, 14, 22, 33, 28, 24, 60, 6, 73, 113, 5, 107, 78, 4, 13, - 8, 45, 72, 74, 46, 104, 63, 71, 44, 87, 70, 103, 92, 57, 49, 7, 88, 68, 77, - 62, 10, 93, 2, 65, 55, 43, 94, 96, 1, 11, 99, 91, 61, 51, 15, 64, 97, 89, 101, - 108, 80, 86, 90, 54, 95, 102, 47, 100, 79, 83, 48, 12, 81, 82, 58, 50, 56, 84, - 85, 53, 52, 59) - } - - def _get_similar_ranking_params(self, cutoff_amount=10000, cutoff_time=2000): - """ - Default values from template_word_similarity_solr.cfg - """ - return { - 'cutoff_amount': cutoff_amount, - 'cutoff_time_ms': cutoff_time, - 'find_similar_to_recid': { - 'more_results_factor': 5, - 'mlt_fl': 'mlt', - 'mlt_mintf': 0, - 'mlt_mindf': 0, - 'mlt_minwl': 0, - 'mlt_maxwl': 0, - 'mlt_maxqt': 25, - 'mlt_maxntp': 1000, - 'mlt_boost': 'false' - } - } - _all_records = get_collection_reclist(CFG_SITE_NAME) + def setUp(self): + init_Solr() + @nottest def test_get_similar_ranked(self): """solrutils - similar results""" - all_ranked = 0 - similar_top = self._get_similar_topN(all_ranked) - recid = 30 - self.assertEqual(similar_top[recid], self._get_similar_result_sequence(recid=recid)) - recid = 59 - self.assertEqual(similar_top[recid], self._get_similar_result_sequence(recid=recid)) - - @nottest - def test_get_similar_ranked_top(self): - """solrutils - similar top results""" - top_n = 5 - similar_top = self._get_similar_topN(top_n) - recid = 30 - self.assertEqual(similar_top[recid], self._get_similar_result_sequence(recid=recid, rows=top_n)) - recid = 59 - self.assertEqual(similar_top[recid], self._get_similar_result_sequence(recid=recid, rows=top_n)) + self.assertEqual(self.get_similar_result_sequence(1), (5, 4, 7, 8, 3, 6, 2, 10, 1)) + self.assertEqual(self.get_similar_result_sequence(8), (3, 6, 9, 7, 2, 4, 5, 1, 10, 8)) class TestSolrWebSearch(InvenioTestCase): @@ -242,31 +160,24 @@ class TestSolrWebSearch(InvenioTestCase): make install-solrutils CFG_SOLR_URL set fulltext index in idxINDEX containing 'SOLR' in indexer column - AND EITHER - Solr index built: ./bibindex -w fulltext for all records - OR - WRD method referring to Solr: /etc/bibrank$ cp template_word_similarity_solr.cfg wrd.cfg - and ./bibrank -w wrd for all records """ + def setUp(self): + init_Solr() @nottest def test_get_result(self): """solrutils - web search results""" self.assertEqual([], test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3AWillnotfind&rg=100', - expected_text="[]")) - - self.assertEqual([], - test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3Ahiggs&rg=100', - expected_text="[12, 47, 48, 51, 52, 55, 56, 58, 68, 79, 80, 81, 85, 89, 96]")) + expected_text='[]')) self.assertEqual([], test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3Aof&rg=100', - expected_text="[8, 10, 11, 12, 15, 43, 44, 45, 46, 47, 48, 49, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 64, 68, 74, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97]")) + expected_text='[1, 2, 7, 8, 10]')) self.assertEqual([], - test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3A%22higgs+boson%22&rg=100', - expected_text="[12, 47, 51, 55, 56, 68, 81, 85]")) + test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3A%22of+the%22&rg=100', + expected_text='[1, 2, 7, 8, 10]')) class TestSolrWebRanking(InvenioTestCase): @@ -274,42 +185,25 @@ class TestSolrWebRanking(InvenioTestCase): make install-solrutils CFG_SOLR_URL set fulltext index in idxINDEX containing 'SOLR' in indexer column - AND EITHER - Solr index built: ./bibindex -w fulltext for all records - OR - WRD method referring to Solr: /etc/bibrank$ cp template_word_similarity_solr.cfg wrd.cfg - and ./bibrank -w wrd for all records + WRD method referring to Solr: /etc/bibrank$ cp template_word_similarity_solr.cfg wrd.cfg """ + def setUp(self): + init_Solr() @nottest def test_get_ranked(self): """solrutils - web ranking results""" self.assertEqual([], test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3AWillnotfind&rg=100&rm=wrd', - expected_text="[]")) + expected_text='[]')) - self.assertEqual([], - test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3Ahiggs&rm=wrd', - expected_text="[12, 51, 79, 80, 81, 55, 47, 56, 96, 58, 68, 52, 48, 89, 85]")) - - self.assertEqual([], - test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3Ahiggs&rg=100&rm=wrd', - expected_text="[12, 80, 81, 79, 51, 55, 47, 56, 96, 58, 68, 52, 48, 89, 85]")) - - # Record 77 is restricted self.assertEqual([], test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3Aof&rm=wrd', - expected_text="[8, 10, 15, 43, 44, 45, 46, 48, 49, 51, 52, 53, 54, 55, 56, 57, 58, 60, 61, 62, 64, 68, 74, 78, 79, 81, 82, 83, 84, 85, 88, 89, 90, 91, 92, 95, 96, 97, 86, 11, 80, 93, 77, 12, 59, 87, 47, 94]", - username='admin')) + expected_text='[8, 2, 1, 10, 7]')) self.assertEqual([], - test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3Aof&rg=100&rm=wrd', - expected_text="[61, 60, 54, 56, 53, 10, 68, 44, 57, 83, 95, 92, 91, 74, 45, 48, 62, 82, 49, 51, 89, 90, 96, 43, 8, 64, 97, 15, 85, 78, 46, 55, 79, 84, 88, 81, 52, 58, 86, 11, 80, 93, 77, 12, 59, 87, 47, 94]", - username='admin')) - - self.assertEqual([], - test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3A%22higgs+boson%22&rg=100&rm=wrd', - expected_text="[12, 47, 51, 68, 81, 85, 55, 56]")) + test_web_page_content(CFG_SITE_URL + '/search?of=id&p=fulltext%3A%22of+the%22&rg=100&rm=wrd', + expected_text='[8, 10, 1, 2, 7]')) class TestSolrWebSimilarToRecid(InvenioTestCase): @@ -318,19 +212,20 @@ class TestSolrWebSimilarToRecid(InvenioTestCase): CFG_SOLR_URL set fulltext index in idxINDEX containing 'SOLR' in indexer column WRD method referring to Solr: /etc/bibrank$ cp template_word_similarity_solr.cfg wrd.cfg - ./bibrank -w wrd for all records """ + def setUp(self): + init_Solr() @nottest def test_get_similar_ranked(self): """solrutils - web similar results""" self.assertEqual([], - test_web_page_content(CFG_SITE_URL + '/search?of=id&p=recid%3A30&rm=wrd', - expected_text="[1, 3, 4, 8, 9, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 31, 34, 43, 44, 49, 50, 56, 58, 61, 64, 66, 67, 69, 71, 73, 75, 76, 77, 78, 82, 85, 86, 87, 89, 90, 95, 96, 98, 104, 107, 109, 113, 65, 62, 60, 47, 46, 100, 99, 102, 91, 80, 7, 5, 92, 88, 74, 57, 55, 108, 84, 81, 79, 54, 101, 11, 103, 94, 48, 83, 72, 63, 2, 68, 51, 53, 97, 93, 70, 45, 52, 14, 59, 6, 10, 32, 33, 29, 30]")) + test_web_page_content(CFG_SITE_URL + '/search?of=id&p=recid%3A1&rm=wrd', + expected_text='[9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 107, 108, 109, 113, 5, 4, 7, 8, 3, 6, 2, 10, 1]')) self.assertEqual([], - test_web_page_content(CFG_SITE_URL + '/search?of=id&p=recid%3A30&rg=100&rm=wrd', - expected_text="[3, 4, 8, 9, 13, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 31, 34, 43, 49, 56, 66, 67, 69, 71, 73, 75, 76, 87, 90, 98, 104, 107, 109, 113, 12, 95, 85, 82, 44, 1, 89, 64, 58, 15, 96, 61, 50, 86, 78, 77, 65, 62, 60, 47, 46, 100, 99, 102, 91, 80, 7, 5, 92, 88, 74, 57, 55, 108, 84, 81, 79, 54, 101, 11, 103, 94, 48, 83, 72, 63, 2, 68, 51, 53, 97, 93, 70, 45, 52, 14, 59, 6, 10, 32, 33, 29, 30]")) + test_web_page_content(CFG_SITE_URL + '/search?of=id&p=recid%3A8&rg=100&rm=wrd', + expected_text='[11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 107, 108, 109, 113, 3, 6, 9, 7, 2, 4, 5, 1, 10, 8]')) class TestSolrLoadLogicalFieldSettings(InvenioTestCase): @@ -339,7 +234,6 @@ class TestSolrLoadLogicalFieldSettings(InvenioTestCase): CFG_SOLR_URL set WRD method referring to Solr: /etc/bibrank$ cp template_word_similarity_solr.cfg wrd.cfg """ - @nottest def test_load_logical_fields(self): """solrutils - load logical fields""" @@ -359,7 +253,6 @@ class TestSolrBuildFieldContent(InvenioTestCase): CFG_SOLR_URL set WRD method referring to Solr: /etc/bibrank$ cp template_word_similarity_solr.cfg wrd.cfg """ - @nottest def test_build_default_field_content(self): """solrutils - build default field content""" @@ -382,7 +275,6 @@ def test_build_custom_field_content(self): self.assertEqual(u"""In 1962, CERN hosted the 11th International Conference on High Energy Physics. Among the distinguished visitors were eight Nobel prizewinners.Left to right: Cecil F. Powell, Isidor I. Rabi, Werner Heisenberg, Edwin M. McMillan, Emile Segre, Tsung Dao Lee, Chen Ning Yang and Robert Hofstadter. En 1962, le CERN est l'hote de la onzieme Conference Internationale de Physique des Hautes Energies. Parmi les visiteurs eminents se trouvaient huit laureats du prix Nobel.De gauche a droite: Cecil F. Powell, Isidor I. Rabi, Werner Heisenberg, Edwin M. McMillan, Emile Segre, Tsung Dao Lee, Chen Ning Yang et Robert Hofstadter.""", get_field_content_in_utf8(6, 'abstract', tags)) - TESTS = [] From a337815b318724dc5498a5f0f75fa6c7fa084d24 Mon Sep 17 00:00:00 2001 From: Patrick Glauner Date: Mon, 5 Aug 2013 19:06:19 +0200 Subject: [PATCH 04/59] WebSearch: new fulltext by default search * Adds checkbox to search terms also in to fulltext to the add to search interface. Signed-off-by: Patrick Glauner --- modules/websearch/lib/search_engine.py | 21 ++++++++++++- modules/websearch/lib/websearch_templates.py | 32 +++++++++++++++++--- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/modules/websearch/lib/search_engine.py b/modules/websearch/lib/search_engine.py index 73a77522a7..180b29486b 100644 --- a/modules/websearch/lib/search_engine.py +++ b/modules/websearch/lib/search_engine.py @@ -6267,9 +6267,28 @@ def prs_simple_search(results_in_any_collection, kwargs=None, req=None, of=None, # recommendations when there are results only in the hosted collections. Also added the if clause to avoid # searching in case we know we only have actual or potential hosted collections results if not only_hosted_colls_actual_or_potential_results_p: - results_in_any_collection.union_update(search_pattern_parenthesised(req, p, f, ap=ap, of=of, verbose=verbose, ln=ln, + # Full-text by default in add to search + if kwargs['aas'] == 2 and f=='fulltext': + # Displays nearest terms box only for the latter part to avoid display of box for metadata in case + # there are only results in the fulltext + results_in_any_collection.union_update(search_pattern_parenthesised(req, p, '', ap=ap, of=of, verbose=verbose, ln=ln, + display_nearest_terms_box=False, + wl=wl)) + # Change pattern to get results from fulltext per term + p_no_fields = re.sub('[a-z]*:', '', p) + p_no_fields = p_no_fields.replace('+', '|') + p_no_fields = p_no_fields.replace('AND', 'OR') + if verbose: + write_warning('Search stage 3: p is "%s"' % p, req=req) + write_warning('Search stage 3: p without fields for fulltext by default search is "%s"' % p_no_fields, req=req) + results_in_any_collection.union_update(search_pattern_parenthesised(req, p_no_fields, 'fulltext', ap=ap, of=of, verbose=verbose, ln=ln, + display_nearest_terms_box=not hosted_colls_actual_or_potential_results_p, + wl=wl)) + else: + results_in_any_collection.union_update(search_pattern_parenthesised(req, p, f, ap=ap, of=of, verbose=verbose, ln=ln, display_nearest_terms_box=not hosted_colls_actual_or_potential_results_p, wl=wl)) + except: register_exception(req=req, alert_admin=True) if of.startswith("h"): diff --git a/modules/websearch/lib/websearch_templates.py b/modules/websearch/lib/websearch_templates.py index f2a36e2135..c13a0c395b 100644 --- a/modules/websearch/lib/websearch_templates.py +++ b/modules/websearch/lib/websearch_templates.py @@ -1159,7 +1159,7 @@ def tmpl_searchfor_addtosearch(self, ln, collection_id, record_count, searchwith return out - def create_addtosearch_box(self, ln, p, p1, searchwithin, header, adv_search_link): + def create_addtosearch_box(self, ln, p, p1, searchwithin, header, adv_search_link, f=''): """ Creates the Search and the Add-to-Search box. @@ -1249,6 +1249,8 @@ def create_addtosearch_box(self, ln, p, p1, searchwithin, header, adv_search_lin %(searchwithin1)s +
+ %(fulltext)s @@ -1262,8 +1264,9 @@ def create_addtosearch_box(self, ln, p, p1, searchwithin, header, adv_search_lin 'matchbox1' : self.tmpl_matchtype_box('m1', '', ln=ln), 'sizepattern' : CFG_WEBSEARCH_ADVANCEDSEARCH_PATTERN_BOX_WIDTH, 'searchwithin1' : searchwithin, - 'add_to_search' : _("Add to Search") - } + 'add_to_search' : _("Add to Search"), + 'fulltext': self.tmpl_fulltext(f, ln='en'), + } return out @@ -1348,6 +1351,26 @@ def tmpl_andornot_box(self, name='op', value='', ln='en'): } return out + def tmpl_fulltext(self, f, ln='en'): + """ + Returns HTML code for the search in fulltext checkbox. + + Parameters: + + - 'f' *string* - The value of the f paramater + + - 'ln' *string* - the language to display + """ + + _ = gettext_set_language(ln) + + checked = 'unchecked' + if f == 'fulltext': + checked = 'checked' + + return """ %(text)s""" % {'checked': checked, + 'text': _('Search also in the full-text of all documents')} + def tmpl_inputdate(self, name, ln, sy=0, sm=0, sd=0): """ Produces *From Date*, *Until Date* kind of selection box. Suitable for search options. @@ -2167,10 +2190,11 @@ def tmpl_search_box(self, ln, aas, cc, cc_intl, ot, sp, selected = '', values = self._add_mark_to_field(value=f1, fields=fieldslist, ln=ln) ) + adv_search_link = create_html_link(self.build_search_url(rm=rm, aas=1, cc=cc, jrec=jrec, ln=ln, rg=rg), {}, _("Advanced Search")) - out += self.create_addtosearch_box(ln, p, p1, searchwithin, '', adv_search_link) + out += self.create_addtosearch_box(ln, p, p1, searchwithin, '', adv_search_link, f) elif aas == 1: # print Advanced Search form: From 97f3fcd21833004d76671b7abf205ed3dc1bf75e Mon Sep 17 00:00:00 2001 From: Patrick Glauner Date: Mon, 26 Aug 2013 15:39:24 +0200 Subject: [PATCH 05/59] BibRank: Solr: better ranking for fulltext by default * Ranks fulltext and metadata properly Signed-off-by: Patrick Glauner --- modules/bibrank/lib/bibrank_record_sorter.py | 4 ++-- modules/miscutil/lib/solrutils_bibrank_searcher.py | 5 ++++- modules/websearch/lib/search_engine.py | 11 ++++++----- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/modules/bibrank/lib/bibrank_record_sorter.py b/modules/bibrank/lib/bibrank_record_sorter.py index 05baa868d5..c9e71400ba 100644 --- a/modules/bibrank/lib/bibrank_record_sorter.py +++ b/modules/bibrank/lib/bibrank_record_sorter.py @@ -222,7 +222,7 @@ def citation(rank_method_code, related_to, hitset, rank_limit_relevance, verbose return rank_by_citations(hits, verbose) -def rank_records(rank_method_code, rank_limit_relevance, hitset, related_to=[], verbose=0, field='', rg=None, jrec=None): +def rank_records(rank_method_code, rank_limit_relevance, hitset, related_to=[], verbose=0, field='', rg=None, jrec=None, kwargs={}): """Sorts given records or related records according to given method Parameters: @@ -293,7 +293,7 @@ def rank_records(rank_method_code, rank_limit_relevance, hitset, related_to=[], if function == "word_similarity_solr": if verbose > 0: voutput += "In Solr part:
" - result = word_similarity_solr(related_to, hitset, METHODS[rank_method_code], verbose, field, ranked_result_amount) + result = word_similarity_solr(related_to, hitset, METHODS[rank_method_code], verbose, field, ranked_result_amount, kwargs) if function == "word_similarity_xapian": if verbose > 0: voutput += "In Xapian part:
" diff --git a/modules/miscutil/lib/solrutils_bibrank_searcher.py b/modules/miscutil/lib/solrutils_bibrank_searcher.py index 28aa607467..eec7c3a92e 100644 --- a/modules/miscutil/lib/solrutils_bibrank_searcher.py +++ b/modules/miscutil/lib/solrutils_bibrank_searcher.py @@ -115,7 +115,7 @@ def get_normalized_ranking_scores(response, hitset_filter = None, recids = []): return (ranked_result, matched_recs) -def word_similarity_solr(pattern, hitset, params, verbose, explicit_field, ranked_result_amount): +def word_similarity_solr(pattern, hitset, params, verbose, explicit_field, ranked_result_amount, kwargs={}): """ Ranking a records containing specified words and returns a sorted list. input: @@ -138,6 +138,9 @@ def word_similarity_solr(pattern, hitset, params, verbose, explicit_field, ranke if pattern: pattern = " ".join(map(str, pattern)) from invenio.search_engine import create_basic_search_units + # Rank global index for fulltext by default in add to search + if kwargs.get('aas', 0) == 2 and explicit_field == 'fulltext': + explicit_field = '' search_units = create_basic_search_units(None, pattern, explicit_field) else: return (None, "Records not ranked. The query is not detailed enough, or not enough records found, for ranking to be possible.", "", voutput) diff --git a/modules/websearch/lib/search_engine.py b/modules/websearch/lib/search_engine.py index 180b29486b..538ee3d9a3 100644 --- a/modules/websearch/lib/search_engine.py +++ b/modules/websearch/lib/search_engine.py @@ -4172,7 +4172,7 @@ def get_tags_from_sort_fields(sort_fields): return tags, '' -def rank_records(req, rank_method_code, rank_limit_relevance, hitset_global, pattern=None, verbose=0, sort_order='d', of='hb', ln=CFG_SITE_LANG, rg=None, jrec=None, field='', sorting_methods=SORTING_METHODS): +def rank_records(req, rank_method_code, rank_limit_relevance, hitset_global, pattern=None, verbose=0, sort_order='d', of='hb', ln=CFG_SITE_LANG, rg=None, jrec=None, field='', sorting_methods=SORTING_METHODS, kwargs={}): """Initial entry point for ranking records, acts like a dispatcher. (i) rank_method_code is in bsrMETHOD, bibsort buckets can be used; (ii)rank_method_code is not in bsrMETHOD, use bibrank; @@ -4206,7 +4206,8 @@ def rank_records(req, rank_method_code, rank_limit_relevance, hitset_global, pat field=field, related_to=related_to, rg=rg, - jrec=jrec) + jrec=jrec, + kwargs=kwargs) # Solution recs can be None, in case of error or other cases # which should be all be changed to return an empty list. @@ -6511,7 +6512,7 @@ def prs_print_records(kwargs=None, results_final=None, req=None, of=None, cc=Non results_final_recIDs_ranked, results_final_relevances, results_final_relevances_prologue, results_final_relevances_epilogue, results_final_comments = \ rank_records(req, rm, 0, results_final[coll], string.split(p) + string.split(p1) + - string.split(p2) + string.split(p3), verbose, so, of, ln, rg, jrec, kwargs['f']) + string.split(p2) + string.split(p3), verbose, so, of, ln, rg, jrec, kwargs['f'], kwargs=kwargs) if of.startswith("h"): write_warning(results_final_comments, req=req) if results_final_recIDs_ranked: @@ -6836,7 +6837,7 @@ def prs_display_results(kwargs=None, results_final=None, req=None, of=None, sf=N if rm: # do we have to rank? results_final_for_all_colls_rank_records_output = rank_records(req, rm, 0, results_final_for_all_selected_colls, p.split() + p1.split() + - p2.split() + p3.split(), verbose, so, of, ln, kwargs['rg'], kwargs['jrec'], kwargs['f']) + p2.split() + p3.split(), verbose, so, of, ln, kwargs['rg'], kwargs['jrec'], kwargs['f'], kwargs=kwargs) if results_final_for_all_colls_rank_records_output[0]: recIDs = results_final_for_all_colls_rank_records_output[0] elif sf or (CFG_BIBSORT_ENABLED and SORTING_METHODS): # do we have to sort? @@ -6891,7 +6892,7 @@ def prs_rank_results(kwargs=None, results_final=None, req=None, colls_to_search= if rm: # do we have to rank? results_final_for_all_colls_rank_records_output = rank_records(req, rm, 0, results_final_for_all_selected_colls, p.split() + p1.split() + - p2.split() + p3.split(), verbose, so, of, field=kwargs['f']) + p2.split() + p3.split(), verbose, so, of, field=kwargs['f'], kwargs=kwargs) if results_final_for_all_colls_rank_records_output[0]: recIDs = results_final_for_all_colls_rank_records_output[0] elif sf or (CFG_BIBSORT_ENABLED and SORTING_METHODS): # do we have to sort? From aaaf54bd963a6ff3d2e364108f240d1a305d818c Mon Sep 17 00:00:00 2001 From: Patrick Glauner Date: Mon, 26 Aug 2013 15:41:39 +0200 Subject: [PATCH 06/59] Solrutils: more accurate ranking scores * Ceils ranking scores to avoid anomalies Signed-off-by: Patrick Glauner --- modules/miscutil/lib/solrutils_bibrank_searcher.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/miscutil/lib/solrutils_bibrank_searcher.py b/modules/miscutil/lib/solrutils_bibrank_searcher.py index eec7c3a92e..0ac8ea2bab 100644 --- a/modules/miscutil/lib/solrutils_bibrank_searcher.py +++ b/modules/miscutil/lib/solrutils_bibrank_searcher.py @@ -25,6 +25,7 @@ import itertools +from math import ceil from invenio.config import CFG_SOLR_URL from invenio.intbitset import intbitset from invenio.errorlib import register_exception @@ -106,7 +107,11 @@ def get_normalized_ranking_scores(response, hitset_filter = None, recids = []): if (not hitset_filter and hitset_filter != []) or recid in hitset_filter or recid in recids: normalised_score = 0 if max_score > 0: - normalised_score = int(100.0 / max_score * float(hit['score'])) + # Ceil score, in particular beneficial for scores in (0,1) and (99,100) to take 1 and 100 + normalised_score = int(ceil(100.0 / max_score * float(hit['score']))) + # Correct possible rounding error + if normalised_score > 100: + normalised_score = 100 ranked_result.append((recid, normalised_score)) matched_recs.add(recid) From bba3e1f66743ff68e8a0d4e1c1d838b61897b732 Mon Sep 17 00:00:00 2001 From: Patrick Glauner Date: Fri, 26 Jul 2013 20:35:26 +0200 Subject: [PATCH 07/59] BibRecord: new MARC code filter * Filters a record based on MARC code. * Supports wildcards in identifiers and subfield code. Signed-off-by: Patrick Glauner --- modules/bibrecord/lib/Makefile.am | 1 + modules/bibrecord/lib/bibrecord.py | 81 ++++++++++ .../lib/bibrecord_regression_tests.py | 144 ++++++++++++++++++ modules/bibrecord/lib/bibrecord_unit_tests.py | 43 +++++- 4 files changed, 266 insertions(+), 3 deletions(-) create mode 100644 modules/bibrecord/lib/bibrecord_regression_tests.py diff --git a/modules/bibrecord/lib/Makefile.am b/modules/bibrecord/lib/Makefile.am index 5096a9f93f..ec938875c8 100644 --- a/modules/bibrecord/lib/Makefile.am +++ b/modules/bibrecord/lib/Makefile.am @@ -20,6 +20,7 @@ pylibdir = $(libdir)/python/invenio pylib_DATA = bibrecord_config.py \ bibrecord.py \ bibrecord_unit_tests.py \ + bibrecord_regression_tests.py \ xmlmarc2textmarc.py \ textmarc2xmlmarc.py diff --git a/modules/bibrecord/lib/bibrecord.py b/modules/bibrecord/lib/bibrecord.py index 546501aa7e..a4830488bb 100644 --- a/modules/bibrecord/lib/bibrecord.py +++ b/modules/bibrecord/lib/bibrecord.py @@ -55,6 +55,8 @@ # has been deleted. CFG_BIBRECORD_KEEP_SINGLETONS = True +CONTROL_TAGS = ('001', '003', '005', '006', '007', '008') + try: import pyRXP if 'pyrxp' in CFG_BIBRECORD_PARSERS_AVAILABLE: @@ -1465,6 +1467,85 @@ def _validate_record_field_positions_global(record): return ("Duplicate global field position '%d' in tag '%s'" % (field[4], tag)) +def get_marc_tag_extended_with_wildcards(tag): + """ + Adds wildcards to a non-control field tag, identifiers and subfieldcode and returns it. + Example: + 001 -> 001 + 595 -> 595%%% + 5955 -> 5955%% + 59555 -> 59555% + 59555a -> 59555a + """ + length = len(tag) + if length == 3 and tag not in CONTROL_TAGS: + return '%s%%%%%%' % tag + if length == 4: + return '%s%%%%' % tag + if length == 5: + return '%s%%' % tag + return tag + +def get_filtered_record(rec, marc_codes): + """ + Filters a record based on MARC code and returns a new filtered record. + Supports wildcards in ind1, ind2, subfieldcode. + Returns entire record if filter list is empty. + """ + if not marc_codes: + return rec + + res = {} + split_tags = map(_get_split_marc_code, marc_codes) + + for tag, ind1, ind2, subfieldcode in split_tags: + # Tag must exist + if tag in rec: + # Control field + if tag in CONTROL_TAGS: + value = record_get_field_value(rec, tag) + record_add_field(res, tag, ind1, ind2, controlfield_value=value) + # Simple subfield + elif '%' not in (ind1, ind2, subfieldcode): + values = record_get_field_values(rec, tag, ind1, ind2, subfieldcode) + for value in values: + record_add_field(res, tag, ind1, ind2, subfields=[(subfieldcode, value)]) + # Wildcard in ind1, ind2 or subfield + elif '%' in (ind1, ind2, subfieldcode): + field_instances = record_get_field_instances(rec, tag, ind1, ind2) + for entity in field_instances: + subfields = [] + if subfieldcode == '%': + subfields = entity[0] + else: + for subfield in entity[0]: + if subfield[0] == subfieldcode: + subfields.append(subfield) + if len(subfields): + record_add_field(res, tag, entity[1], entity[2], subfields=subfields) + + return res + +def _get_split_marc_code(marc_code): + """ + Splits a MARC code into tag, ind1 ind2, and subfieldcode. + Accepts '_' values which are converted to ' '. + """ + tag, ind1, ind2, subfieldcode = '', '', '', '' + + length = len(marc_code) + + if length >= 3: + tag = marc_code[0:3] + if length >= 4: + ind1 = marc_code[3].replace('_', '') + if length >= 5: + ind2 = marc_code[4].replace('_', '') + if length == 6: + subfieldcode = marc_code[5].replace('_', '') + return tag, ind1, ind2, subfieldcode + + def _record_sort_by_indicators(record): """Sorts the fields inside the record by indicators.""" for tag, fields in record.items(): diff --git a/modules/bibrecord/lib/bibrecord_regression_tests.py b/modules/bibrecord/lib/bibrecord_regression_tests.py new file mode 100644 index 0000000000..54a1833023 --- /dev/null +++ b/modules/bibrecord/lib/bibrecord_regression_tests.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +## +## This file is part of Invenio. +## Copyright (C) 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +""" +The BibRecord regression test suite. +""" + +import unittest + +from invenio.config import CFG_SITE_URL, \ + CFG_SITE_RECORD +from invenio import bibrecord +from invenio.testutils import make_test_suite, run_test_suite +from invenio.search_engine import get_record + + +class BibRecordFilterBibrecordTest(unittest.TestCase): + """ bibrecord - testing for code filtering""" + + def setUp(self): + self.rec = get_record(10) + + def test_empty_filter(self): + """bibrecord - empty filter""" + self.assertEqual(bibrecord.get_filtered_record(self.rec, []), self.rec) + + def test_filter_tag_only(self): + """bibrecord - filtering only by MARC tag""" + # Exist + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['001']), {'001': [([], ' ', ' ', '10', 1)]}) + # Do not exist + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['037']), {}) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['856']), {}) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['999']), {}) + # Sequence + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['001', '999']), {'001': [([], ' ', ' ', '10', 1)]}) + # Some tags do not exist + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['001', '260', '856', '400', '500', '999']), {'001': [([], ' ', ' ', '10', 1)]}) + + def test_filter_subfields(self): + """bibrecord - filtering subfields""" + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['65017a']), {'650': [([('a', 'Particle Physics - Experimental Results')], '1', '7', '', 1)],}) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['65017a', '650172']), {'650': [([('a', 'Particle Physics - Experimental Results')], '1', '7', '', 1), + ([('2', 'SzGeCERN')], '1', '7', '', 2)]}) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['8560_f']), {'856': [([('f', 'valerie.brunner@cern.ch')], '0', ' ', '', 1)]}) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['260__a']), {'260': [([('a', 'Geneva')], ' ', ' ', '', 1)],}) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['595__a']), {'595': [([('a', 'CERN EDS')], ' ', ' ', '', 1), + ([('a', '20011220SLAC')], ' ', ' ', '', 2), + ([('a', 'giva')], ' ', ' ', '', 3), + ([('a', 'LANL EDS')], ' ', ' ', '', 4)]}) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['8564_u']), {'856': [([('u', '%s/%s/10/files/ep-2001-094.ps.gz' % (CFG_SITE_URL, CFG_SITE_RECORD))], '4', ' ', '', 1), + ([('u', '%s/%s/10/files/ep-2001-094.pdf' % (CFG_SITE_URL, CFG_SITE_RECORD))], '4', ' ', '', 2)]}) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['595__a', '8564_u']), {'595': [([('a', 'CERN EDS')], ' ', ' ', '', 1), + ([('a', '20011220SLAC')], ' ', ' ', '', 2), + ([('a', 'giva')], ' ', ' ', '', 3), + ([('a', 'LANL EDS')], ' ', ' ', '', 4)], + '856': [([('u', '%s/%s/10/files/ep-2001-094.ps.gz' % (CFG_SITE_URL, CFG_SITE_RECORD))], '4', ' ', '', 5), + ([('u', '%s/%s/10/files/ep-2001-094.pdf' % (CFG_SITE_URL, CFG_SITE_RECORD))], '4', ' ', '', 6)]}) + + def test_filter_comprehensive(self): + """bibrecord - comprehensive filtering""" + tags = ['001', '035', '037__a', '65017a', '650'] + res = {} + res['001'] = [([], ' ', ' ', '10', 1)] + res['037'] = [([('a', 'hep-ex/0201013')], ' ', ' ', '', 2)] + res['650'] = [([('a', 'Particle Physics - Experimental Results')], '1', '7', '', 3)] + self.assertEqual(bibrecord.get_filtered_record(self.rec, tags), res) + + def test_filter_wildcards(self): + """bibrecord - wildcards filtering""" + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['595__%']), {'595': [([('a', 'CERN EDS')], ' ', ' ', '', 1), + ([('a', '20011220SLAC')], ' ', ' ', '', 2), + ([('a', 'giva')], ' ', ' ', '', 3), + ([('a', 'LANL EDS')], ' ', ' ', '', 4)]}) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['909CS%']), {'909': [([('s', 'n'), ('w', '200231')], 'C', 'S', '', 1)]}) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['856%']), {}) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['856%_u']), {'856': [([('u', '%s/%s/10/files/ep-2001-094.ps.gz' % (CFG_SITE_URL, CFG_SITE_RECORD))], '4', ' ', '', 1), + ([('u', '%s/%s/10/files/ep-2001-094.pdf' % (CFG_SITE_URL, CFG_SITE_RECORD))], '4', ' ', '', 2)]}) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['909%5v']), {}) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['909%5b']), {'909': [([('b', 'CER')], 'C', '5', '', 1)]}) + + def test_filter_multi_wildcards(self): + """bibrecord - multi wildcards filtering""" + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['909%%_']), {}) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['856%_%']), {'856': [([('f', 'valerie.brunner@cern.ch')], '0', ' ', '', 1), + ([('s', '217223'), ('u', '%s/%s/10/files/ep-2001-094.ps.gz' % (CFG_SITE_URL, CFG_SITE_RECORD))], '4', ' ', '', 2), + ([('s', '383040'), ('u', '%s/%s/10/files/ep-2001-094.pdf' % (CFG_SITE_URL, CFG_SITE_RECORD))], '4', ' ', '', 3)]}) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['909%%b']), {'909': [([('b', '11')], 'C', '0', '', 1), + ([('b', 'CER')], 'C', '5', '', 2)]}) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['909%%%']), {'909': [([('y', '2002')], 'C', '0', '', 1), + ([('e', 'ALEPH')], 'C', '0', '', 2), + ([('b', '11')], 'C', '0', '', 3), + ([('p', 'EP')], 'C', '0', '', 4), + ([('a', 'CERN LEP')], 'C', '0', '', 5), + ([('c', '2001-12-19'), ('l', '50'), ('m', '2002-02-19'), ('o', 'BATCH')], 'C', '1', '', 6), + ([('u', 'CERN')], 'C', '1', '', 7), + ([('p', 'Eur. Phys. J., C')], 'C', '4', '', 8), + ([('b', 'CER')], 'C', '5', '', 9), + ([('s', 'n'), ('w', '200231')], 'C', 'S', '', 10), + ([('o', 'oai:cds.cern.ch:CERN-EP-2001-094'), ('p', 'cern:experiment')], 'C', 'O', '', 11)]}) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['980%%%']), bibrecord.get_filtered_record(self.rec, ['980_%%'])) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['980_%%']), bibrecord.get_filtered_record(self.rec, ['980%_%'])) + self.assertEqual(bibrecord.get_filtered_record(self.rec, ['980__%']), bibrecord.get_filtered_record(self.rec, ['980%%%'])) + + def test_filter_wildcard_comprehensive(self): + """bibrecord - comprehensive wildcard filtering""" + tags = ['595__%', '909CS%', '856%', '856%_%', '909%5b', '980%%%'] + res = {} + res['595'] = [([('a', 'CERN EDS')], ' ', ' ', '', 1), + ([('a', '20011220SLAC')], ' ', ' ', '', 2), + ([('a', 'giva')], ' ', ' ', '', 3), + ([('a', 'LANL EDS')], ' ', ' ', '', 4)] + res['856'] = [([('f', 'valerie.brunner@cern.ch')], '0', ' ', '', 5), + ([('s', '217223'), ('u', '%s/%s/10/files/ep-2001-094.ps.gz' % (CFG_SITE_URL, CFG_SITE_RECORD))], '4', ' ', '', 6), + ([('s', '383040'), ('u', '%s/%s/10/files/ep-2001-094.pdf' % (CFG_SITE_URL, CFG_SITE_RECORD))], '4', ' ', '', 7)] + res['909'] = [([('s', 'n'), ('w', '200231')], 'C', 'S', '', 8), + ([('b', 'CER')], 'C', '5', '', 9)] + res['980'] = [([('a', 'PREPRINT')], ' ', ' ', '', 10), + ([('a', 'ALEPHPAPER')], ' ', ' ', '', 11)] + self.assertEqual(bibrecord.get_filtered_record(self.rec, tags), res) + + +TEST_SUITE = make_test_suite( + BibRecordFilterBibrecordTest, + ) + +if __name__ == '__main__': + run_test_suite(TEST_SUITE, warn_user=True) diff --git a/modules/bibrecord/lib/bibrecord_unit_tests.py b/modules/bibrecord/lib/bibrecord_unit_tests.py index 5c985c4e8a..e50065d321 100644 --- a/modules/bibrecord/lib/bibrecord_unit_tests.py +++ b/modules/bibrecord/lib/bibrecord_unit_tests.py @@ -18,15 +18,18 @@ # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """ -The BibRecord test suite. +The BibRecord unit test suite. """ from invenio.testutils import InvenioTestCase from invenio.config import CFG_TMPDIR, \ - CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG + CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG, \ + CFG_SITE_URL, \ + CFG_SITE_RECORD from invenio import bibrecord, bibrecord_config from invenio.testutils import make_test_suite, run_test_suite +from invenio.search_engine import get_record try: import pyRXP @@ -1770,6 +1773,38 @@ def test_extract_oai_id(self): 'oai:atlantis:1') +class BibRecordSplitMARCCodeTest(InvenioTestCase): + """ bibrecord - testing for MARC code spliting""" + + def test_split_tag(self): + """bibrecord - splitting MARC code""" + self.assertEqual(bibrecord._get_split_marc_code('001'), ('001', '', '', '')) + self.assertEqual(bibrecord._get_split_marc_code('65017a'), ('650', '1', '7', 'a')) + self.assertEqual(bibrecord._get_split_marc_code('037__a'), ('037', '', '', 'a')) + self.assertEqual(bibrecord._get_split_marc_code('8560_f'), ('856', '0', '', 'f')) + self.assertEqual(bibrecord._get_split_marc_code('909C1'), ('909', 'C', '1', '')) + self.assertEqual(bibrecord._get_split_marc_code('650'), ('650', '', '', '')) + self.assertEqual(bibrecord._get_split_marc_code('650__%'), ('650', '', '', '%')) + self.assertEqual(bibrecord._get_split_marc_code('650%'), ('650', '%', '', '')) + + +class BibRecordExtensionWithWildcardsTagTest(InvenioTestCase): + """ bibrecord - testing for MARC tag extension with wildcards""" + + def test_split_tag(self): + """bibrecord - extending MARC tag with wildcards""" + # No valid tag + self.assertEqual(bibrecord.get_marc_tag_extended_with_wildcards('01'), '01') + # Control tag + self.assertEqual(bibrecord.get_marc_tag_extended_with_wildcards('001'), '001') + # Extending tag + self.assertEqual(bibrecord.get_marc_tag_extended_with_wildcards('595'), '595%%%') + # Extending tag and identifier(s) + self.assertEqual(bibrecord.get_marc_tag_extended_with_wildcards('0001'), '0001%%') + self.assertEqual(bibrecord.get_marc_tag_extended_with_wildcards('00012'), '00012%') + self.assertEqual(bibrecord.get_marc_tag_extended_with_wildcards('00012a'), '00012a') + + TEST_SUITE = make_test_suite( BibRecordSuccessTest, BibRecordParsersTest, @@ -1795,7 +1830,9 @@ def test_extract_oai_id(self): BibRecordSingletonTest, BibRecordNumCharRefTest, BibRecordExtractIdentifiersTest, - BibRecordDropDuplicateFieldsTest + BibRecordDropDuplicateFieldsTest, + BibRecordSplitMARCCodeTest, + BibRecordExtensionWithWildcardsTagTest, ) if __name__ == '__main__': From 3e330248ec356cebca1328ae9d93c8d0dce95068 Mon Sep 17 00:00:00 2001 From: Patrick Glauner Date: Mon, 29 Jul 2013 18:03:54 +0200 Subject: [PATCH 08/59] BibFormat: new 'xmf' format * Supports filtering per MARC tag, identifier and subfield. Signed-off-by: Patrick Glauner --- .../etc/format_templates/MARCXML_FIELDED.bft | 3 +++ .../etc/format_templates/Makefile.am | 1 + .../bibformat/etc/output_formats/Makefile.am | 1 + modules/bibformat/etc/output_formats/XMF.bfo | 1 + modules/bibformat/lib/bibformat.py | 17 ++++++++----- modules/bibformat/lib/bibformat_engine.py | 24 ++++++++++++------- .../bibformat/lib/elements/bfe_xml_record.py | 17 +++++++++++-- modules/websearch/lib/search_engine.py | 5 ++-- 8 files changed, 50 insertions(+), 19 deletions(-) create mode 100644 modules/bibformat/etc/format_templates/MARCXML_FIELDED.bft create mode 100644 modules/bibformat/etc/output_formats/XMF.bfo diff --git a/modules/bibformat/etc/format_templates/MARCXML_FIELDED.bft b/modules/bibformat/etc/format_templates/MARCXML_FIELDED.bft new file mode 100644 index 0000000000..4ac0175212 --- /dev/null +++ b/modules/bibformat/etc/format_templates/MARCXML_FIELDED.bft @@ -0,0 +1,3 @@ +MARC XML +Standard MARC XML output + diff --git a/modules/bibformat/etc/format_templates/Makefile.am b/modules/bibformat/etc/format_templates/Makefile.am index 5ef4d5b554..a8ebc1e9f2 100644 --- a/modules/bibformat/etc/format_templates/Makefile.am +++ b/modules/bibformat/etc/format_templates/Makefile.am @@ -51,6 +51,7 @@ etc_DATA = Default_HTML_captions.bft \ DataCite.xsl \ Default_Mobile_brief.bft \ Default_Mobile_detailed.bft \ + MARCXML_FIELDED.bft \ Authority_HTML_brief.bft \ People_HTML_detailed.bft \ People_HTML_brief.bft \ diff --git a/modules/bibformat/etc/output_formats/Makefile.am b/modules/bibformat/etc/output_formats/Makefile.am index 2878a3ed0c..041d742b3f 100644 --- a/modules/bibformat/etc/output_formats/Makefile.am +++ b/modules/bibformat/etc/output_formats/Makefile.am @@ -23,6 +23,7 @@ etc_DATA = HB.bfo \ HP.bfo \ HX.bfo \ XM.bfo \ + XMF.bfo \ EXCEL.bfo \ XP.bfo \ XN.bfo \ diff --git a/modules/bibformat/etc/output_formats/XMF.bfo b/modules/bibformat/etc/output_formats/XMF.bfo new file mode 100644 index 0000000000..1aa575d89e --- /dev/null +++ b/modules/bibformat/etc/output_formats/XMF.bfo @@ -0,0 +1 @@ +default: MARCXML_FIELDED.bft diff --git a/modules/bibformat/lib/bibformat.py b/modules/bibformat/lib/bibformat.py index 257a62d34c..058ba0a5bb 100644 --- a/modules/bibformat/lib/bibformat.py +++ b/modules/bibformat/lib/bibformat.py @@ -52,7 +52,7 @@ # def format_record(recID, of, ln=CFG_SITE_LANG, verbose=0, search_pattern=None, xml_record=None, user_info=None, on_the_fly=False, - save_missing=True, force_2nd_pass=False): + save_missing=True, force_2nd_pass=False, ot=''): """ Returns the formatted record with id 'recID' and format 'of' @@ -80,6 +80,8 @@ def format_record(recID, of, ln=CFG_SITE_LANG, verbose=0, search_pattern=None, @param recID: the id of the record to fetch @param of: the output format code + @param ot: output only these MARC tags (e.g. ['100', '999']), only supported for 'xmf' format + @type ot: list @return: formatted record as String, or '' if it does not exist """ out, needs_2nd_pass = bibformat_engine.format_record_1st_pass( @@ -91,7 +93,8 @@ def format_record(recID, of, ln=CFG_SITE_LANG, verbose=0, search_pattern=None, xml_record=xml_record, user_info=user_info, on_the_fly=on_the_fly, - save_missing=save_missing) + save_missing=save_missing, + ot=ot) if needs_2nd_pass or force_2nd_pass: out = bibformat_engine.format_record_2nd_pass( recID=recID, @@ -101,7 +104,8 @@ def format_record(recID, of, ln=CFG_SITE_LANG, verbose=0, search_pattern=None, verbose=verbose, search_pattern=search_pattern, xml_record=xml_record, - user_info=user_info) + user_info=user_info, + ot=ot) return out @@ -138,7 +142,7 @@ def record_get_xml(recID, format='xm', decompress=zlib.decompress): def format_records(recIDs, of, ln=CFG_SITE_LANG, verbose=0, search_pattern=None, xml_records=None, user_info=None, record_prefix=None, record_separator=None, record_suffix=None, prologue="", - epilogue="", req=None, on_the_fly=False): + epilogue="", req=None, on_the_fly=False, ot=''): """ Format records given by a list of record IDs or a list of records as xml. Adds a prefix before each record, a suffix after each @@ -195,11 +199,12 @@ def format_records(recIDs, of, ln=CFG_SITE_LANG, verbose=0, search_pattern=None, @param req: an optional request object where to print records @param on_the_fly: if False, try to return an already preformatted version of the record in the database @type on_the_fly: boolean + @param ot: output only these MARC tags (e.g. "100,700,909C0b"), only supported for 'xmf' format + @type ot: string @rtype: string """ if req is not None: req.write(prologue) - formatted_records = '' #Fill one of the lists with Nones @@ -229,7 +234,7 @@ def format_records(recIDs, of, ln=CFG_SITE_LANG, verbose=0, search_pattern=None, #Print formatted record formatted_record = format_record(recIDs[i], of, ln, verbose, search_pattern, xml_records[i], - user_info, on_the_fly) + user_info, on_the_fly, ot=ot) formatted_records += formatted_record if req is not None: req.write(formatted_record) diff --git a/modules/bibformat/lib/bibformat_engine.py b/modules/bibformat/lib/bibformat_engine.py index fb98d5d33c..776b2369ff 100644 --- a/modules/bibformat/lib/bibformat_engine.py +++ b/modules/bibformat/lib/bibformat_engine.py @@ -197,7 +197,7 @@ def format_record(recID, of, ln=CFG_SITE_LANG, verbose=0, - search_pattern=None, xml_record=None, user_info=None): + search_pattern=None, xml_record=None, user_info=None, ot=''): """ Formats a record given output format. Main entry function of bibformat engine. @@ -224,6 +224,8 @@ def format_record(recID, of, ln=CFG_SITE_LANG, verbose=0, @param search_pattern: list of strings representing the user request in web interface @param xml_record: an xml string representing the record to format @param user_info: the information of the user who will view the formatted page + @param ot: output only these MARC tags (e.g. "100,700,909C0b"), only supported for 'xmf' format + @type ot: string @return: formatted record """ if search_pattern is None: @@ -239,7 +241,7 @@ def format_record(recID, of, ln=CFG_SITE_LANG, verbose=0, # But if format not found for new BibFormat, then call old BibFormat #Create a BibFormat Object to pass that contain record and context - bfo = BibFormatObject(recID, ln, search_pattern, xml_record, user_info, of) + bfo = BibFormatObject(recID, ln, search_pattern, xml_record, user_info, of, ot) if of.lower() != 'xm' and (not bfo.get_record() or record_empty(bfo.get_record())): @@ -290,7 +292,7 @@ def format_record(recID, of, ln=CFG_SITE_LANG, verbose=0, def format_record_1st_pass(recID, of, ln=CFG_SITE_LANG, verbose=0, search_pattern=None, xml_record=None, user_info=None, on_the_fly=False, - save_missing=True): + save_missing=True, ot=''): """ Format a record in given output format. @@ -362,7 +364,7 @@ def format_record_1st_pass(recID, of, ln=CFG_SITE_LANG, verbose=0, out += """\n
Found preformatted output for record %i (cache updated on %s).
""" % (recID, last_updated) - if of.lower() == 'xm': + if of.lower() in ('xm', 'xmf'): res = filter_hidden_fields(res, user_info) # try to replace language links in pre-cached res, if applicable: if ln != CFG_SITE_LANG and of.lower() in CFG_BIBFORMAT_DISABLE_I18N_FOR_CACHED_FORMATS: @@ -396,10 +398,11 @@ def format_record_1st_pass(recID, of, ln=CFG_SITE_LANG, verbose=0, verbose=verbose, search_pattern=search_pattern, xml_record=xml_record, - user_info=user_info) + user_info=user_info, + ot=ot) out += out_ - if of.lower() in ('xm', 'xoaimarc'): + if of.lower() in ('xm', 'xoaimarc', 'xmf'): out = filter_hidden_fields(out, user_info, force_filtering=of.lower()=='xoaimarc') # We have spent time computing this format @@ -443,9 +446,9 @@ def format_record_1st_pass(recID, of, ln=CFG_SITE_LANG, verbose=0, def format_record_2nd_pass(recID, template, ln=CFG_SITE_LANG, search_pattern=None, xml_record=None, - user_info=None, of=None, verbose=0): + user_info=None, of=None, verbose=0, ot=''): # Create light bfo object - bfo = BibFormatObject(recID, ln, search_pattern, xml_record, user_info, of) + bfo = BibFormatObject(recID, ln, search_pattern, xml_record, user_info, of, ot) # Translations template = translate_template(template, ln) # Format template @@ -1825,7 +1828,7 @@ class BibFormatObject(object): req = None # DEPRECATED: use bfo.user_info instead. Used by WebJournal. def __init__(self, recID, ln=CFG_SITE_LANG, search_pattern=None, - xml_record=None, user_info=None, output_format=''): + xml_record=None, user_info=None, output_format='', ot=''): """ Creates a new bibformat object, with given record. @@ -1855,6 +1858,8 @@ def __init__(self, recID, ln=CFG_SITE_LANG, search_pattern=None, @param xml_record: a xml string of the record to format @param user_info: the information of the user who will view the formatted page @param output_format: the output_format used for formatting this record + @param ot: output only these MARC tags (e.g. ['100', '999']), only supported for 'xmf' format + @type ot: list """ self.xml_record = None # *Must* remain empty if recid is given if xml_record is not None: @@ -1872,6 +1877,7 @@ def __init__(self, recID, ln=CFG_SITE_LANG, search_pattern=None, self.user_info = user_info if self.user_info is None: self.user_info = collect_user_info(None) + self.ot = ot def get_record(self): """ diff --git a/modules/bibformat/lib/elements/bfe_xml_record.py b/modules/bibformat/lib/elements/bfe_xml_record.py index 020987a902..e4c6c40d37 100644 --- a/modules/bibformat/lib/elements/bfe_xml_record.py +++ b/modules/bibformat/lib/elements/bfe_xml_record.py @@ -24,14 +24,27 @@ def format_element(bfo, type='xml', encodeForXML='yes'): """ Prints the complete current record as XML. - @param type: the type of xml. Can be 'xml', 'oai_dc', 'marcxml', 'xd' + @param type: the type of xml. Can be 'xml', 'oai_dc', 'marcxml', 'xd', 'xmf' @param encodeForXML: if 'yes', replace all < > and & with html corresponding escaped characters. """ from invenio.bibformat_utils import record_get_xml + from invenio.bibrecord import get_filtered_record, record_xml_output, get_marc_tag_extended_with_wildcards from invenio.textutils import encode_for_xml + from invenio.search_engine import get_record #Can be used to output various xml flavours. - out = record_get_xml(bfo.recID, format=type, on_the_fly=True) + out = '' + if type == 'xmf': + tags = bfo.ot + if tags != ['']: + filter_tags = map(get_marc_tag_extended_with_wildcards, tags) + else: + filter_tags = [] + record = get_record(bfo.recID) + filtered_record = get_filtered_record(record, filter_tags) + out = record_xml_output(filtered_record) + else: + out = record_get_xml(bfo.recID, format=type, on_the_fly=True) if encodeForXML.lower() == 'yes': return encode_for_xml(out) diff --git a/modules/websearch/lib/search_engine.py b/modules/websearch/lib/search_engine.py index 73a77522a7..3f6ae7a78d 100644 --- a/modules/websearch/lib/search_engine.py +++ b/modules/websearch/lib/search_engine.py @@ -4560,7 +4560,7 @@ def print_records(req, recIDs, jrec=1, rg=CFG_WEBSEARCH_DEF_RECORDS_IN_GROUPS, f if print_records_prologue_p: print_records_prologue(req, format) - if ot: + if ot and not format.startswith('xmf'): # asked to print some filtered fields only, so call print_record() on the fly: for recid in recIDs: x = print_record(recid, @@ -4584,7 +4584,8 @@ def print_records(req, recIDs, jrec=1, rg=CFG_WEBSEARCH_DEF_RECORDS_IN_GROUPS, f search_pattern=search_pattern, record_separator="\n", user_info=user_info, - req=req) + req=req, + ot=ot) # print footer if needed if print_records_epilogue_p: From eacb83bf2ffcbae77a7634fbae274c3d744a16db Mon Sep 17 00:00:00 2001 From: Joe MacMahon Date: Fri, 17 Oct 2014 17:42:12 +0200 Subject: [PATCH 09/59] stats: Elasticsearch logging with Lumberjack. * Added disabled-by-default config options for Elasticsearch logging. * When Elasticsearch logging is configured, don't populate rnkPAGEVIEWS and rnkDOWNLOADS. Signed-off-by: Joe MacMahon --- config/invenio.conf | 44 ++++++++++++ modules/bibdocfile/lib/bibdocfile.py | 41 ++++++++--- .../bibdocfile/lib/bibdocfile_webinterface.py | 2 +- .../lib/bibrank_downloads_similarity.py | 33 +++++++-- modules/miscutil/lib/Makefile.am | 3 +- modules/miscutil/lib/__init__.py | 4 ++ modules/miscutil/lib/elasticsearch_logging.py | 71 +++++++++++++++++++ modules/miscutil/lib/inveniocfg.py | 4 +- modules/websearch/lib/search_engine.py | 3 +- requirements.txt | 1 + 10 files changed, 188 insertions(+), 18 deletions(-) create mode 100644 modules/miscutil/lib/elasticsearch_logging.py diff --git a/config/invenio.conf b/config/invenio.conf index 408411ba6a..0675c64832 100644 --- a/config/invenio.conf +++ b/config/invenio.conf @@ -2565,6 +2565,50 @@ CFG_ARXIV_URL_PATTERN = http://export.arxiv.org/pdf/%sv%s.pdf # e.g. CFG_REDIS_HOSTS = [{'db': 0, 'host': '127.0.0.1', 'port': 7001}] CFG_REDIS_HOSTS = {'default': [{'db': 0, 'host': '127.0.0.1', 'port': 6379}]} +################################# +## Elasticsearch Configuration ## +################################# + +## CFG_ELASTICSEARCH_LOGGING -- Whether to use Elasticsearch logging or not +CFG_ELASTICSEARCH_LOGGING = 0 + +## CFG_ELASTICSEARCH_INDEX_PREFIX -- The prefix to be used for the +## Elasticsearch indices. +CFG_ELASTICSEARCH_INDEX_PREFIX = invenio- + +## CFG_ELASTICSEARCH_HOSTS -- The list of Elasticsearch hosts to connect to. +## This is a list of dictionaries with connection information. +CFG_ELASTICSEARCH_HOSTS = [{'host': '127.0.0.1', 'port': 9200}] + +## CFG_ELASTICSEARCH_SUFFIX_FORMAT -- The time format string to base the +## suffixes for the Elasticsearch indices on. E.g. "%Y.%m" for indices to be +## called "invenio-2014.10" for example. +CFG_ELASTICSEARCH_SUFFIX_FORMAT = %Y.%m + +## CFG_ELASTICSEARCH_MAX_QUEUE_LENGTH -- The maximum length the queue of events +## is allowed to grow to before it is flushed to Elasticsearch. If you don't +## want to set a maximum, and rely entirely on the periodic flush instead, set +## this to -1. +CFG_ELASTICSEARCH_MAX_QUEUE_LENGTH = -1 + +## CFG_ELASTICSEARCH_FLUSH_INTERVAL -- The time (in seconds) to wait between +## flushes of the event queue to Elasticsearch. If you want to disable +## periodic flushing and instead rely on the max. queue length to trigger +## flushes, set this to -1. +CFG_ELASTICSEARCH_FLUSH_INTERVAL = 30 + +## CFG_ELASTICSEARCH_BOT_AGENT_STRINGS -- A list of strings which, if found in +## the user agent string, will cause a 'bot' flag to be added to the logged +## event. This list taken from bots marked "active" at +## . Googlebot and +## bingbot added to the head of the list for speed. +CFG_ELASTICSEARCH_BOT_AGENT_STRINGS = ['Googlebot', 'bingbot', 'Arachnoidea', +'FAST-WebCrawler', 'Fluffy the spider', 'Gigabot', 'Gulper', 'ia_archiver', +'MantraAgent', 'MSN', 'Scooter', 'Scrubby', 'Slurp', 'Teoma_agent1', 'Winona', +'ZyBorg', 'Almaden', 'Cyveillance', 'DTSearch', 'Girafa.com', 'Indy Library', +'LinkWalker', 'MarkWatch', 'NameProtect', 'Robozilla', 'Teradex Mapper', +'Tracerlock', 'W3C_Validator', 'WDG_Validator', 'Zealbot'] + ########################## # THAT's ALL, FOLKS! ## ########################## diff --git a/modules/bibdocfile/lib/bibdocfile.py b/modules/bibdocfile/lib/bibdocfile.py index 0cada52da5..596b3b2526 100644 --- a/modules/bibdocfile/lib/bibdocfile.py +++ b/modules/bibdocfile/lib/bibdocfile.py @@ -122,7 +122,9 @@ CFG_BIBDOCFILE_ENABLE_BIBDOCFSINFO_CACHE, \ CFG_BIBDOCFILE_ADDITIONAL_KNOWN_MIMETYPES, \ CFG_BIBDOCFILE_PREFERRED_MIMETYPES_MAPPING, \ - CFG_BIBCATALOG_SYSTEM + CFG_BIBCATALOG_SYSTEM, \ + CFG_ELASTICSEARCH_LOGGING, \ + CFG_ELASTICSEARCH_BOT_AGENT_STRINGS from invenio.bibcatalog import BIBCATALOG_SYSTEM from invenio.bibdocfile_config import CFG_BIBDOCFILE_ICON_SUBFORMAT_RE, \ CFG_BIBDOCFILE_DEFAULT_ICON_SUBFORMAT, CFG_BIBDOCFILE_STREAM_ARCHIVE_FORMATS @@ -130,6 +132,11 @@ import invenio.template +if CFG_ELASTICSEARCH_LOGGING: + import logging + + _DOWNLOAD_LOG = logging.getLogger('events.downloads') + def _plugin_bldr(dummy, plugin_code): """Preparing the plugin dictionary structure""" ret = {} @@ -2900,7 +2907,8 @@ def get_file_number(self): """Return the total number of files.""" return len(self.docfiles) - def register_download(self, ip_address, version, docformat, userid=0, recid=0): + def register_download(self, ip_address, version, docformat, user_agent, + userid=0, recid=0): """Register the information about a download of a particular file.""" docformat = normalize_format(docformat) @@ -2909,12 +2917,29 @@ def register_download(self, ip_address, version, docformat, userid=0, recid=0): docformat = docformat.upper() if not version: version = self.get_latest_version() - return run_sql("INSERT INTO rnkDOWNLOADS " - "(id_bibrec,id_bibdoc,file_version,file_format," - "id_user,client_host,download_time) VALUES " - "(%s,%s,%s,%s,%s,INET_ATON(%s),NOW())", - (recid, self.id, version, docformat, - userid, ip_address,)) + if CFG_ELASTICSEARCH_LOGGING: + log_entry = { + 'id_bibrec': recid, + 'id_bibdoc': self.id, + 'file_version': version, + 'file_format': docformat, + 'id_user': userid, + 'client_host': ip_address, + 'user_agent': user_agent + } + if user_agent is not None: + for bot in CFG_ELASTICSEARCH_BOT_AGENT_STRINGS: + if bot in user_agent: + log_entry['bot'] = True + break + _DOWNLOAD_LOG.info(log_entry) + else: + return run_sql("INSERT INTO rnkDOWNLOADS " + "(id_bibrec,id_bibdoc,file_version,file_format," + "id_user,client_host,download_time) VALUES " + "(%s,%s,%s,%s,%s,INET_ATON(%s),NOW())", + (recid, self.id, version, docformat, + userid, ip_address,)) def get_incoming_relations(self, rel_type=None): """Return all relations in which this BibDoc appears on target position diff --git a/modules/bibdocfile/lib/bibdocfile_webinterface.py b/modules/bibdocfile/lib/bibdocfile_webinterface.py index 198b72fb17..bd9d1fdf39 100644 --- a/modules/bibdocfile/lib/bibdocfile_webinterface.py +++ b/modules/bibdocfile/lib/bibdocfile_webinterface.py @@ -225,7 +225,7 @@ def getfile(req, form): if not docfile.hidden_p(): if not readonly: ip = str(req.remote_ip) - doc.register_download(ip, docfile.get_version(), docformat, uid, self.recid) + doc.register_download(ip, docfile.get_version(), docformat, req.headers_in.get('User-Agent'), uid, self.recid) try: return docfile.stream(req, download=is_download) except InvenioBibDocFileError, msg: diff --git a/modules/bibrank/lib/bibrank_downloads_similarity.py b/modules/bibrank/lib/bibrank_downloads_similarity.py index d79ff4f33d..568cd0261b 100644 --- a/modules/bibrank/lib/bibrank_downloads_similarity.py +++ b/modules/bibrank/lib/bibrank_downloads_similarity.py @@ -22,11 +22,18 @@ from invenio.config import \ CFG_ACCESS_CONTROL_LEVEL_SITE, \ - CFG_CERN_SITE + CFG_CERN_SITE, \ + CFG_ELASTICSEARCH_LOGGING, \ + CFG_ELASTICSEARCH_BOT_AGENT_STRINGS from invenio.dbquery import run_sql from invenio.bibrank_downloads_indexer import database_tuples_to_single_list from invenio.search_engine_utils import get_fieldvalues +if CFG_ELASTICSEARCH_LOGGING: + import logging + + _PAGEVIEW_LOG = logging.getLogger('events.pageviews') + def record_exists(recID): """Return 1 if record RECID exists. Return 0 if it doesn't exist. @@ -46,7 +53,7 @@ def record_exists(recID): ### INTERFACE -def register_page_view_event(recid, uid, client_ip_address): +def register_page_view_event(recid, uid, client_ip_address, user_agent): """Register Detailed record page view event for record RECID consulted by user UID from machine CLIENT_HOST_IP. To be called by the search engine. @@ -55,10 +62,24 @@ def register_page_view_event(recid, uid, client_ip_address): # do not register access if we are in read-only access control # site mode: return [] - return run_sql("INSERT INTO rnkPAGEVIEWS " \ - " (id_bibrec,id_user,client_host,view_time) " \ - " VALUES (%s,%s,INET_ATON(%s),NOW())", \ - (recid, uid, client_ip_address)) + if CFG_ELASTICSEARCH_LOGGING: + log_event = { + 'id_bibrec': recid, + 'id_user': uid, + 'client_host': client_ip_address, + 'user_agent': user_agent + } + if user_agent is not None: + for bot in CFG_ELASTICSEARCH_BOT_AGENT_STRINGS: + if bot in user_agent: + log_event['bot'] = True + break + _PAGEVIEW_LOG.info(log_event) + else: + return run_sql("INSERT INTO rnkPAGEVIEWS " \ + " (id_bibrec,id_user,client_host,view_time) " \ + " VALUES (%s,%s,INET_ATON(%s),NOW())", \ + (recid, uid, client_ip_address)) def calculate_reading_similarity_list(recid, type="pageviews"): """Calculate reading similarity data to use in reading similarity diff --git a/modules/miscutil/lib/Makefile.am b/modules/miscutil/lib/Makefile.am index 2d094d2711..ac0feb64a7 100644 --- a/modules/miscutil/lib/Makefile.am +++ b/modules/miscutil/lib/Makefile.am @@ -102,7 +102,8 @@ pylib_DATA = __init__.py \ hepdatautils_unit_tests.py \ filedownloadutils.py \ filedownloadutils_unit_tests.py \ - viafutils.py + viafutils.py \ + elasticsearch_logging.py jsdir=$(localstatedir)/www/js diff --git a/modules/miscutil/lib/__init__.py b/modules/miscutil/lib/__init__.py index e6d1852fa9..baa980c8e8 100644 --- a/modules/miscutil/lib/__init__.py +++ b/modules/miscutil/lib/__init__.py @@ -24,3 +24,7 @@ import sys reload(sys) sys.setdefaultencoding('utf8') + +## Because we use getLogger calls to do logging, handlers aren't initialised +## unless we explicitly call/import this code somewhere. +import elasticsearch_logging diff --git a/modules/miscutil/lib/elasticsearch_logging.py b/modules/miscutil/lib/elasticsearch_logging.py new file mode 100644 index 0000000000..be7ab306f9 --- /dev/null +++ b/modules/miscutil/lib/elasticsearch_logging.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +## +## This file is part of Invenio. +## Copyright (C) 2014 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +__revision__ = \ + "$Id$" + +from invenio.config import \ + CFG_ELASTICSEARCH_LOGGING, \ + CFG_ELASTICSEARCH_INDEX_PREFIX, \ + CFG_ELASTICSEARCH_HOSTS, \ + CFG_ELASTICSEARCH_SUFFIX_FORMAT, \ + CFG_ELASTICSEARCH_MAX_QUEUE_LENGTH, \ + CFG_ELASTICSEARCH_FLUSH_INTERVAL + +if CFG_ELASTICSEARCH_LOGGING: + import lumberjack + import logging + import sys + +def initialise_lumberjack(): + if not CFG_ELASTICSEARCH_LOGGING: + return None + config = lumberjack.get_default_config() + config['index_prefix'] = CFG_ELASTICSEARCH_INDEX_PREFIX + + if CFG_ELASTICSEARCH_MAX_QUEUE_LENGTH == -1: + config['max_queue_length'] = None + else: + config['max_queue_length'] = CFG_ELASTICSEARCH_MAX_QUEUE_LENGTH + + if CFG_ELASTICSEARCH_FLUSH_INTERVAL == -1: + config['interval'] = None + else: + config['interval'] = CFG_ELASTICSEARCH_FLUSH_INTERVAL + + lj = lumberjack.Lumberjack( + hosts=CFG_ELASTICSEARCH_HOSTS, + config=config) + + handler = lj.get_handler(suffix_format=CFG_ELASTICSEARCH_SUFFIX_FORMAT) + logging.getLogger('events').addHandler(handler) + logging.getLogger('events').setLevel(logging.INFO) + + logging.getLogger('lumberjack').addHandler( + logging.StreamHandler(sys.stderr)) + logging.getLogger('lumberjack').setLevel(logging.ERROR) + + return lj + +LUMBERJACK = initialise_lumberjack() + +def register_schema(*args, **kwargs): + if not CFG_ELASTICSEARCH_LOGGING: + return None + return LUMBERJACK.register_schema(*args, **kwargs) diff --git a/modules/miscutil/lib/inveniocfg.py b/modules/miscutil/lib/inveniocfg.py index 45132b5f0d..fecb52115c 100644 --- a/modules/miscutil/lib/inveniocfg.py +++ b/modules/miscutil/lib/inveniocfg.py @@ -189,7 +189,9 @@ def convert_conf_option(option_name, option_value): 'CFG_REDIS_HOSTS', 'CFG_BIBSCHED_INCOMPATIBLE_TASKS', 'CFG_ICON_CREATION_FORMAT_MAPPINGS', - 'CFG_BIBEDIT_AUTOCOMPLETE']: + 'CFG_BIBEDIT_AUTOCOMPLETE', + 'CFG_ELASTICSEARCH_HOSTS', + 'CFG_ELASTICSEARCH_BOT_AGENT_STRINGS']: try: option_value = option_value[1:-1] if option_name == "CFG_BIBEDIT_EXTEND_RECORD_WITH_COLLECTION_TEMPLATE" and option_value.strip().startswith("{"): diff --git a/modules/websearch/lib/search_engine.py b/modules/websearch/lib/search_engine.py index 73a77522a7..f3086069f5 100644 --- a/modules/websearch/lib/search_engine.py +++ b/modules/websearch/lib/search_engine.py @@ -5885,7 +5885,8 @@ def prs_detailed_record(kwargs=None, req=None, of=None, cc=None, aas=None, ln=No so=so, sp=sp, rm=rm, em=em, nb_found=len(range(recid, recidb))) if req and of.startswith("h"): # register detailed record page view event client_ip_address = str(req.remote_ip) - register_page_view_event(recid, uid, client_ip_address) + register_page_view_event(recid, uid, client_ip_address, + req.get_headers_in().get('User-Agent')) else: # record does not exist if of == "id": return [] diff --git a/requirements.txt b/requirements.txt index 6f17065ccc..f1e215ec33 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,3 +29,4 @@ redis==2.9.0 nydus==0.10.6 Cerberus==0.5 matplotlib +-e git+git://github.com/CERNDocumentServer/lumberjack.git#egg=lumberjack From 0c4262738fcf37e024ca479ae172e1d362fe7873 Mon Sep 17 00:00:00 2001 From: Harris Tzovanakis Date: Thu, 4 Dec 2014 15:21:09 +0100 Subject: [PATCH 10/59] WebStat: register custom events on es * Adds `loanrequest` schema for `Lumberjack`. * Adds custom events elastic search logging. Signed-off-by: Harris Tzovanakis --- modules/webstat/lib/webstat.py | 48 ++++++++++++++++++++++----- modules/webstat/lib/webstat_config.py | 26 ++++++++++++++- 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/modules/webstat/lib/webstat.py b/modules/webstat/lib/webstat.py index f5e440eea6..7232fc052e 100644 --- a/modules/webstat/lib/webstat.py +++ b/modules/webstat/lib/webstat.py @@ -26,6 +26,7 @@ import calendar from datetime import timedelta from urllib import quote +from logging import getLogger from invenio import template from invenio.config import \ @@ -33,8 +34,11 @@ CFG_TMPDIR, \ CFG_SITE_URL, \ CFG_SITE_LANG, \ - CFG_WEBSTAT_BIBCIRCULATION_START_YEAR -from invenio.webstat_config import CFG_WEBSTAT_CONFIG_PATH + CFG_WEBSTAT_BIBCIRCULATION_START_YEAR, \ + CFG_ELASTICSEARCH_LOGGING +from invenio.webstat_config import \ + CFG_WEBSTAT_CONFIG_PATH, \ + CFG_ELASTICSEARCH_EVENTS_MAP from invenio.bibindex_engine_utils import get_all_indexes from invenio.bibindex_tokenizers.BibIndexJournalTokenizer import CFG_JOURNAL_TAG from invenio.search_engine import get_coll_i18nname, \ @@ -724,6 +728,7 @@ def destroy_customevents(): msg += destroy_customevent(event[0]) return msg + def register_customevent(event_id, *arguments): """ Registers a custom event. Will add to the database's event tables @@ -739,17 +744,25 @@ def register_customevent(event_id, *arguments): @param *arguments: The rest of the parameters of the function call @type *arguments: [params] """ - res = run_sql("SELECT CONCAT('staEVENT', number),cols " + \ - "FROM staEVENT WHERE id = %s", (event_id, )) + query = """ + SELECT CONCAT('staEVENT', number), + cols + FROM staEVENT + WHERE id = %s + """ + params = (event_id,) + res = run_sql(query, params) + # the id does not exist if not res: - return # the id does not exist + return tbl_name = res[0][0] if res[0][1]: col_titles = cPickle.loads(res[0][1]) else: col_titles = [] if len(col_titles) != len(arguments[0]): - return # there is different number of arguments than cols + # there is different number of arguments than cols + return # Make sql query if len(arguments[0]) != 0: @@ -758,18 +771,35 @@ def register_customevent(event_id, *arguments): for title in col_titles: sql_query.append("`%s`" % title) sql_query.append(",") - sql_query.pop() # del the last ',' + # del the last ',' + sql_query.pop() sql_query.append(") VALUES (") for argument in arguments[0]: sql_query.append("%s") sql_query.append(",") sql_param.append(argument) - sql_query.pop() # del the last ',' + # del the last ',' + sql_query.pop() sql_query.append(")") sql_str = ''.join(sql_query) run_sql(sql_str, tuple(sql_param)) + + # Register the event on elastic search + if CFG_ELASTICSEARCH_LOGGING and event_id in \ + CFG_ELASTICSEARCH_EVENTS_MAP.keys(): + # Initialize elastic search handler + elastic_search_parameters = zip(col_titles, arguments[0]) + event_logger_name = "events.{0}".format(event_id) + logger = getLogger(event_logger_name) + log_event = {} + for key, value in elastic_search_parameters: + log_event[key] = value + logger.info(log_event) else: - run_sql("INSERT INTO %s () VALUES ()" % wash_table_column_name(tbl_name)) # kwalitee: disable=sql + # kwalitee: disable=sql + run_sql( + "INSERT INTO %s () VALUES ()" % wash_table_column_name(tbl_name) + ) def cache_keyevent_trend(ids=[]): diff --git a/modules/webstat/lib/webstat_config.py b/modules/webstat/lib/webstat_config.py index 917dd71aac..da93dc732d 100644 --- a/modules/webstat/lib/webstat_config.py +++ b/modules/webstat/lib/webstat_config.py @@ -21,6 +21,30 @@ __revision__ = "$Id$" -from invenio.config import CFG_ETCDIR +from invenio.config import CFG_ETCDIR, CFG_ELASTICSEARCH_LOGGING, CFG_CERN_SITE CFG_WEBSTAT_CONFIG_PATH = CFG_ETCDIR + "/webstat/webstat.cfg" + +if CFG_CERN_SITE: + CFG_ELASTICSEARCH_EVENTS_MAP = { + "events.loanrequest": { + "_source": { + "enabled": True + }, + "properties": { + "request_id": { + "type": "integer" + }, + "load_id": { + "type": "integer" + } + } + } + } +else: + CFG_ELASTICSEARCH_EVENTS_MAP = {} + +if CFG_ELASTICSEARCH_LOGGING: + from invenio.elasticsearch_logging import register_schema + for name, arguments in CFG_ELASTICSEARCH_EVENTS_MAP.items(): + register_schema(name, arguments) From 12c9d1875a64f0b83144bd40fc51180cf95afbf5 Mon Sep 17 00:00:00 2001 From: "Esteban J. G. Gabancho" Date: Mon, 12 Jan 2015 14:11:59 +0100 Subject: [PATCH 11/59] containerutils: new lazy data structures * Back ports lazy dictionaries from next. Signed-off-by: Esteban J. G. Gabancho --- modules/miscutil/lib/containerutils.py | 141 +++++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/modules/miscutil/lib/containerutils.py b/modules/miscutil/lib/containerutils.py index f40cfa2d3e..f439981f46 100644 --- a/modules/miscutil/lib/containerutils.py +++ b/modules/miscutil/lib/containerutils.py @@ -20,6 +20,7 @@ """ import re +from six import iteritems def get_substructure(data, path): """ @@ -274,3 +275,143 @@ def set(self, key, value, extend=False): def update(self, E, **F): self._dict.update(E, **F) + + +class LazyDict(object): + + """Lazy dictionary that evaluates its content when it is first accessed. + + Example: + + .. code-block:: python + + def my_dict(): + from werkzeug.utils import import_string + return {'foo': import_string('foo')} + + lazy_dict = LazyDict(my_dict) + # at this point the internal dictionary is empty + lazy_dict['foo'] + """ + + def __init__(self, function=dict): + """Initialize lazy dictionary with given function. + + :param function: it must return a dictionary like structure + """ + super(LazyDict, self).__init__() + self._cached_dict = None + self._function = function + + def _evaluate_function(self): + self._cached_dict = self._function() + + def __getitem__(self, key): + """Return item from cache if it exists else create it.""" + if self._cached_dict is None: + self._evaluate_function() + return self._cached_dict.__getitem__(key) + + def __setitem__(self, key, value): + """Set item to cache if it exists else create it.""" + if self._cached_dict is None: + self._evaluate_function() + return self._cached_dict.__setitem__(key, value) + + def __delitem__(self, key): + """Delete item from cache if it exists else create it.""" + if self._cached_dict is None: + self._evaluate_function() + return self._cached_dict.__delitem__(key) + + def __getattr__(self, key): + """Get cache attribute if it exists else create it.""" + if self._cached_dict is None: + self._evaluate_function() + return getattr(self._cached_dict, key) + + def __iter__(self): + if self._cached_dict is None: + self._evaluate_function() + return self._cached_dict.__iter__() + + def iteritems(self): + if self._cached_dict is None: + self._evaluate_function() + return iteritems(self._cached_dict) + + def iterkeys(self): + if self._cached_dict is None: + self._evaluate_function() + return self._cached_dict.iterkeys() + + def itervalues(self): + if self._cached_dict is None: + self._evaluate_function() + return self._cached_dict.itervalues() + + def expunge(self): + self._cached_dict = None + + def get(self, key, default=None): + try: + return self.__getitem__(key) + except KeyError: + return default + + +class LaziestDict(LazyDict): + + """Even lazier dictionary (maybe the laziest). + + It does not have content and when a key is accessed it tries to evaluate + only this key. + + Example: + + .. code-block:: python + + def reader_discover(key): + from werkzeug.utils import import_string + return import_string( + 'invenio.jsonalchemy.jsonext.readers%sreader:reader' % (key) + ) + + laziest_dict = LaziestDict(reader_discover) + + laziest_dict['json'] + # It will give you the JsonReader class + """ + + def __init__(self, function=dict): + """Initialize laziest dictionary with given function. + + :param function: it must accept one parameter (the key of the + dictionary) and returns the element which will be store that key. + """ + super(LaziestDict, self).__init__(function) + + def _evaluate_function(self): + """Create empty dict if necessary.""" + if self._cached_dict is None: + self._cached_dict = {} + + def __getitem__(self, key): + if self._cached_dict is None: + self._evaluate_function() + if key not in self._cached_dict: + try: + self._cached_dict.__setitem__(key, self._function(key)) + except: + raise KeyError(key) + return self._cached_dict.__getitem__(key) + + def __contains__(self, key): + if self._cached_dict is None: + self._evaluate_function() + if key not in self._cached_dict: + try: + self.__getitem__(key) + except: + return False + return True From 3c4da44ce8d116b89c8cc987e11e794709cbeb91 Mon Sep 17 00:00:00 2001 From: "Esteban J. G. Gabancho" Date: Wed, 11 Feb 2015 15:56:45 +0100 Subject: [PATCH 12/59] xmlDict: initial release backport Signed-off-by: Esteban J. G. Gabancho --- modules/miscutil/lib/Makefile.am | 1 + modules/miscutil/lib/xmlDict.py | 87 ++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 modules/miscutil/lib/xmlDict.py diff --git a/modules/miscutil/lib/Makefile.am b/modules/miscutil/lib/Makefile.am index 2d094d2711..fdd4cc7704 100644 --- a/modules/miscutil/lib/Makefile.am +++ b/modules/miscutil/lib/Makefile.am @@ -84,6 +84,7 @@ pylib_DATA = __init__.py \ xapianutils_bibrank_indexer.py \ xapianutils_bibrank_searcher.py \ xapianutils_config.py \ + xmlDict.py \ remote_debugger.py \ remote_debugger_config.py \ remote_debugger_wsgi_reload.py \ diff --git a/modules/miscutil/lib/xmlDict.py b/modules/miscutil/lib/xmlDict.py new file mode 100644 index 0000000000..04a17cb9ed --- /dev/null +++ b/modules/miscutil/lib/xmlDict.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# +## The following code is authored by Duncan McGreggor and is licensed +## under PSF license. It was taken from +## . + +import six +import xml.etree.ElementTree as ElementTree + + +class XmlListConfig(list): + def __init__(self, aList): + for element in aList: + if element: + # treat like dict + if len(element) == 1 or element[0].tag != element[1].tag: + self.append(XmlDictConfig(element)) + # treat like list + elif element[0].tag == element[1].tag: + self.append(XmlListConfig(element)) + elif element.text: + text = element.text.strip() + if text: + self.append(text) + + +class XmlDictConfig(dict): + ''' + Example usage: + + >>> tree = ElementTree.parse('your_file.xml') + >>> root = tree.getroot() + >>> xmldict = XmlDictConfig(root) + + Or, if you want to use an XML string: + + >>> root = ElementTree.XML(xml_string) + >>> xmldict = XmlDictConfig(root) + + And then use xmldict for what it is... a dict. + ''' + def __init__(self, parent_element): + if parent_element.items(): + self.update(dict(parent_element.items())) + + for element in parent_element: + if element: + # treat like dict - we assume that if the first two tags + # in a series are different, then they are all different. + if len(element) == 1 or element[0].tag != element[1].tag: + aDict = XmlDictConfig(element) + # treat like list - we assume that if the first two tags + # in a series are the same, then the rest are the same. + else: + # here, we put the list in dictionary; the key is the + # tag name the list elements all share in common, and + # the value is the list itself + aDict = {element[0].tag: XmlListConfig(element)} + # if the tag has attributes, add those to the dict + if element.items(): + aDict.update(dict(element.items())) + self.update({element.tag: aDict}) + # this assumes that if you've got an attribute in a tag, + # you won't be having any text. This may or may not be a + # good idea -- time will tell. It works for the way we are + # currently doing XML configuration files... + elif element.items(): + + # this assumes that if we got a single attribute + # with no children the attribute defines the type of the text + if len(element.items()) == 1 and not list(element): + # check if its str or unicode and if the text is empty, + # otherwise the tag has empty text, no need to add it + if isinstance(element.text, six.string_types) and element.text.strip() != '': + # we have an attribute in the tag that specifies + # most probably the type of the text + tag = element.items()[0][1] + self.update({element.tag: dict({tag: element.text})}) + else: + self.update({element.tag: dict(element.items())}) + if not list(element) and isinstance(element.text, six.string_types)\ + and element.text.strip() != '': + self[element.tag].update(dict({"text": element.text})) + # finally, if there are no child tags and no attributes, extract + # the text + else: + self.update({element.tag: element.text}) From d6eccff348e4582d7dce4a8e35d36d1c4995c421 Mon Sep 17 00:00:00 2001 From: "Esteban J. G. Gabancho" Date: Wed, 11 Feb 2015 16:33:38 +0100 Subject: [PATCH 13/59] pidstore: initial release backport Signed-off-by: Esteban J. G. Gabancho --- configure.ac | 1 + modules/miscutil/lib/Makefile.am | 4 +- modules/miscutil/lib/dataciteutils.py | 177 +++++-- modules/miscutil/lib/inveniocfg.py | 3 +- modules/miscutil/lib/pid_provider.py | 168 +++++++ .../miscutil/lib/pid_providers/Makefile.am | 24 + .../miscutil/lib/pid_providers/__init__.py | 18 + .../miscutil/lib/pid_providers/datacite.py | 203 ++++++++ .../miscutil/lib/pid_providers/local_doi.py | 45 ++ modules/miscutil/lib/pid_store.py | 432 ++++++++++++++++++ .../invenio_2015_01_15_pidstore_initial.py | 90 ++++ modules/miscutil/sql/tabbibclean.sql | 2 + modules/miscutil/sql/tabcreate.sql | 30 ++ 13 files changed, 1166 insertions(+), 31 deletions(-) create mode 100644 modules/miscutil/lib/pid_provider.py create mode 100644 modules/miscutil/lib/pid_providers/Makefile.am create mode 100644 modules/miscutil/lib/pid_providers/__init__.py create mode 100644 modules/miscutil/lib/pid_providers/datacite.py create mode 100644 modules/miscutil/lib/pid_providers/local_doi.py create mode 100644 modules/miscutil/lib/pid_store.py create mode 100644 modules/miscutil/lib/upgrades/invenio_2015_01_15_pidstore_initial.py diff --git a/configure.ac b/configure.ac index f634d004fd..0c2a182e96 100644 --- a/configure.ac +++ b/configure.ac @@ -811,6 +811,7 @@ AC_CONFIG_FILES([config.nice \ modules/miscutil/etc/ckeditor_scientificchar/lang/Makefile \ modules/miscutil/lib/Makefile \ modules/miscutil/lib/upgrades/Makefile \ + modules/miscutil/lib/pid_providers/Makefile \ modules/miscutil/sql/Makefile \ modules/miscutil/web/Makefile \ modules/webaccess/Makefile \ diff --git a/modules/miscutil/lib/Makefile.am b/modules/miscutil/lib/Makefile.am index fdd4cc7704..734f528f2d 100644 --- a/modules/miscutil/lib/Makefile.am +++ b/modules/miscutil/lib/Makefile.am @@ -15,7 +15,7 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. -SUBDIRS = upgrades +SUBDIRS = upgrades pid_providers pylibdir = $(libdir)/python/invenio @@ -65,6 +65,8 @@ pylib_DATA = __init__.py \ pluginutils.py \ pluginutils_unit_tests.py \ redisutils.py \ + pid_provider.py \ + pid_store.py \ plotextractor.py \ plotextractor_converter.py \ plotextractor_getter.py \ diff --git a/modules/miscutil/lib/dataciteutils.py b/modules/miscutil/lib/dataciteutils.py index ef74a5b51d..ea8811487a 100644 --- a/modules/miscutil/lib/dataciteutils.py +++ b/modules/miscutil/lib/dataciteutils.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # This file is part of Invenio. -# Copyright (C) 2012 CERN. +# Copyright (C) 2012, 2015 CERN. # # Invenio is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -31,7 +31,11 @@ Example of usage: doc = ''' - + 10.5072/invenio.test.1 @@ -78,8 +82,12 @@ if not HAS_SSL: from warnings import warn - warn("Module ssl not installed. Please install with e.g. 'pip install ssl'. Required for HTTPS connections to DataCite.", RuntimeWarning) + warn("Module ssl not installed. Please install with e.g. " + "'pip install ssl'. Required for HTTPS connections to DataCite.", + RuntimeWarning) +import re +from invenio.xmlDict import XmlDictConfig, ElementTree # Uncomment to enable debugging of HTTP connection and uncomment line in # DataCiteRequest.request() @@ -93,15 +101,18 @@ # OpenSSL 1.0.0 has a reported bug with SSLv3/TLS handshake. # Python libs affected are httplib2 and urllib2. Eg: # httplib2.SSLHandshakeError: [Errno 1] _ssl.c:497: - # error:14077438:SSL routines:SSL23_GET_SERVER_HELLO:tlsv1 alert internal error - # custom HTTPS opener, banner's oracle 10g server supports SSLv3 only + # error:14077438:SSL routines:SSL23_GET_SERVER_HELLO:tlsv1 alert internal + # error custom HTTPS opener, banner's oracle 10g server supports SSLv3 only class HTTPSConnectionV3(httplib.HTTPSConnection): def __init__(self, *args, **kwargs): httplib.HTTPSConnection.__init__(self, *args, **kwargs) def connect(self): try: - sock = socket.create_connection((self.host, self.port), self.timeout) + sock = socket.create_connection( + (self.host, self.port), + self.timeout + ) except AttributeError: # Python 2.4 compatibility (does not deal with IPv6) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -116,9 +127,15 @@ def connect(self): pass try: - self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv3) + self.sock = ssl.wrap_socket( + sock, self.key_file, self.cert_file, + ssl_version=ssl.PROTOCOL_TLSv1 + ) except ssl.SSLError: - self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23) + self.sock = ssl.wrap_socket( + sock, self.key_file, self.cert_file, + ssl_version=ssl.PROTOCOL_SSLv23 + ) class HTTPSHandlerV3(urllib2.HTTPSHandler): def https_open(self, req): @@ -176,7 +193,10 @@ class DataCiteRequestError(DataCiteError): class DataCiteNoContentError(DataCiteRequestError): - """ DOI is known to MDS, but is not resolvable (might be due to handle's latency) """ + """ + DOI is known to MDS, but is not resolvable (might be due to handle's + latency) + """ pass @@ -235,7 +255,8 @@ class DataCiteRequest(object): query string on all requests. @type default_params: dict """ - def __init__(self, base_url=None, username=None, password=None, default_params={}): + def __init__(self, base_url=None, username=None, password=None, + default_params={}): self.base_url = base_url self.username = username self.password = password @@ -265,7 +286,8 @@ def request(self, url, method='GET', body=None, params={}, headers={}): self.data = None self.code = None - headers['Authorization'] = 'Basic ' + base64.encodestring(self.username + ':' + self.password) + headers['Authorization'] = 'Basic ' + \ + base64.encodestring(self.username + ':' + self.password) if headers['Authorization'][-1] == '\n': headers['Authorization'] = headers['Authorization'][:-1] @@ -284,7 +306,8 @@ def request(self, url, method='GET', body=None, params={}, headers={}): # HTTP client requests must end with double newline (not added # by urllib2) body += '\r\n\r\n' - body = body.encode('utf-8') + if isinstance(body, unicode): + body = body.encode('utf-8') else: if params: url = "%s?%s" % (url, urlencode(params)) @@ -301,10 +324,10 @@ def request(self, url, method='GET', body=None, params={}, headers={}): res = opener.open(request) self.code = res.code self.data = res.read() - except urllib2.HTTPError, e: + except urllib2.HTTPError as e: self.code = e.code self.data = e.msg - except urllib2.URLError, e: + except urllib2.URLError as e: raise HttpError(e) def get(self, url, params={}, headers={}): @@ -313,11 +336,13 @@ def get(self, url, params={}, headers={}): def post(self, url, body=None, params={}, headers={}): """ Make a POST request """ - return self.request(url, method='POST', body=body, params=params, headers=headers) + return self.request(url, method='POST', body=body, params=params, + headers=headers) def delete(self, url, params={}, headers={}): """ Make a DELETE request """ - return self.request(url, method="DELETE", params=params, headers=headers) + return self.request(url, method="DELETE", params=params, + headers=headers) class DataCite(object): @@ -325,10 +350,11 @@ class DataCite(object): DataCite API wrapper """ - def __init__(self, username=None, password=None, url=None, prefix=None, test_mode=None, api_ver="2"): + def __init__(self, username=None, password=None, url=None, prefix=None, + test_mode=None, api_ver="2"): """ - Initialize DataCite API. In case parameters are not specified via keyword - arguments, they will be read from the Invenio configuration. + Initialize DataCite API. In case parameters are not specified via + keyword arguments, they will be read from the Invenio configuration. @param username: DataCite username (or CFG_DATACITE_USERNAME) @type username: str @@ -336,27 +362,37 @@ def __init__(self, username=None, password=None, url=None, prefix=None, test_mod @param password: DataCite password (or CFG_DATACITE_PASSWORD) @type password: str - @param url: DataCite API base URL (or CFG_DATACITE_URL). Defaults to https://mds.datacite.org/. + @param url: DataCite API base URL (or CFG_DATACITE_URL). Defaults to + https://mds.datacite.org/. @type url: str - @param prefix: DOI prefix (or CFG_DATACITE_DOI_PREFIX). Defaults to 10.5072 (DataCite test prefix). + @param prefix: DOI prefix (or CFG_DATACITE_DOI_PREFIX). Defaults to + 10.5072 (DataCite test prefix). @type prefix: str - @param test_mode: Set to True to enable test mode (or CFG_DATACITE_TESTMODE). Defaults to False. + @param test_mode: Set to True to enable test mode (or + CFG_DATACITE_TESTMODE). Defaults to False. @type test_mode: boolean - @param api_ver: DataCite API version. Currently has no effect. Default to 2. + @param api_ver: DataCite API version. Currently has no effect. + Default to 2. @type api_ver: str """ if not HAS_SSL: - warn("Module ssl not installed. Please install with e.g. 'pip install ssl'. Required for HTTPS connections to DataCite.") - - self.username = username or getattr(config, 'CFG_DATACITE_USERNAME', '') - self.password = password or getattr(config, 'CFG_DATACITE_PASSWORD', '') - self.prefix = prefix or getattr(config, 'CFG_DATACITE_DOI_PREFIX', '10.5072') + warn("Module ssl not installed. Please install with e.g. " + "'pip install ssl'. Required for HTTPS connections to " + "DataCite.") + + self.username = username or getattr(config, 'CFG_DATACITE_USERNAME', + '') + self.password = password or getattr(config, 'CFG_DATACITE_PASSWORD', + '') + self.prefix = prefix or getattr(config, 'CFG_DATACITE_DOI_PREFIX', + '10.5072') self.api_ver = api_ver # Currently not used - self.api_url = url or getattr(config, 'CFG_DATACITE_URL', 'https://mds.datacite.org/') + self.api_url = url or getattr(config, 'CFG_DATACITE_URL', + 'https://mds.datacite.org/') if self.api_url[-1] != '/': self.api_url = self.api_url + "/" @@ -535,3 +571,86 @@ def media_post(self, doi, media): return r.data else: raise DataCiteError.factory(r.code) + + +class DataciteMetadata(object): + + def __init__(self, doi): + + self.url = "http://data.datacite.org/application/x-datacite+xml/" + self.error = False + try: + data = urllib2.urlopen(self.url + doi).read() + except urllib2.HTTPError: + self.error = True + + if not self.error: + # Clean the xml for parsing + data = re.sub('<\?xml.*\?>', '', data, count=1) + + # Remove the resource tags + data = re.sub('', '', data) + self.data = '' + \ + data[0:len(data) - 11] + '' + self.root = ElementTree.XML(self.data) + self.xml = XmlDictConfig(self.root) + + def get_creators(self, attribute='creatorName'): + if 'creators' in self.xml: + if isinstance(self.xml['creators']['creator'], list): + return [c[attribute] for c in self.xml['creators']['creator']] + else: + return self.xml['creators']['creator'][attribute] + + return None + + def get_titles(self): + if 'titles' in self.xml: + return self.xml['titles']['title'] + return None + + def get_publisher(self): + if 'publisher' in self.xml: + return self.xml['publisher'] + return None + + def get_dates(self): + if 'dates' in self.xml: + if isinstance(self.xml['dates']['date'], dict): + return self.xml['dates']['date'].values()[0] + return self.xml['dates']['date'] + return None + + def get_publication_year(self): + if 'publicationYear' in self.xml: + return self.xml['publicationYear'] + return None + + def get_language(self): + if 'language' in self.xml: + return self.xml['language'] + return None + + def get_related_identifiers(self): + pass + + def get_description(self, description_type='Abstract'): + if 'descriptions' in self.xml: + if isinstance(self.xml['descriptions']['description'], list): + for description in self.xml['descriptions']['description']: + if description_type in description: + return description[description_type] + elif isinstance(self.xml['descriptions']['description'], dict): + description = self.xml['descriptions']['description'] + if description_type in description: + return description[description_type] + elif len(description) == 1: + # return the only description + return description.values()[0] + + return None + + def get_rights(self): + if 'titles' in self.xml: + return self.xml['rights'] + return None diff --git a/modules/miscutil/lib/inveniocfg.py b/modules/miscutil/lib/inveniocfg.py index 45132b5f0d..c716ec2d9d 100644 --- a/modules/miscutil/lib/inveniocfg.py +++ b/modules/miscutil/lib/inveniocfg.py @@ -245,7 +245,8 @@ def convert_conf_option(option_name, option_value): 'CFG_OAUTH2_PROVIDERS', 'CFG_BIBFORMAT_CACHED_FORMATS', 'CFG_BIBEDIT_ADD_TICKET_RT_QUEUES', - 'CFG_BIBAUTHORID_ENABLED_REMOTE_LOGIN_SYSTEMS',]: + 'CFG_BIBAUTHORID_ENABLED_REMOTE_LOGIN_SYSTEMS', + 'PIDSTORE_OBJECT_TYPES', ]: out = "[" for elem in option_value[1:-1].split(","): if elem: diff --git a/modules/miscutil/lib/pid_provider.py b/modules/miscutil/lib/pid_provider.py new file mode 100644 index 0000000000..fa6c70fa5c --- /dev/null +++ b/modules/miscutil/lib/pid_provider.py @@ -0,0 +1,168 @@ +# -*- coding: utf-8 -*- +## +## This file is part of Invenio. +## Copyright (C) 2015 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +from invenio.containerutils import LazyDict + + +class PidProvider(object): + """ + Abstract class for persistent identifier provider classes. + + Subclasses must implement register, update, delete and is_provider_for_pid + methods and register itself: + + class MyProvider(PidProvider): + pid_type = "mypid" + + def reserve(self, pid, *args, **kwargs): + return True + + def register(self, pid, *args, **kwargs): + return True + + def update(self, pid, *args, **kwargs): + return True + + def delete(self, pid, *args, **kwargs): + try: + ... + except Exception as e: + pid.log("DELETE","Deletion failed") + return False + else: + pid.log("DELETE","Successfully deleted") + return True + + def is_provider_for_pid(self, pid_str): + pass + + PidProvider.register_provider(MyProvider) + + + The provider is responsible for handling of errors, as well as logging of + actions happening to the pid. See example above as well as the + DataCitePidProvider. + + Each method takes variable number of argument and keywords arguments. This + can be used to pass additional information to the provider when registering + a persistent identifier. E.g. a DOI requires URL and metadata to be able + to register the DOI. + """ + + def __load_providers(): + from invenio.pid_store import _PID_PROVIDERS + registry = dict() + for provider in _PID_PROVIDERS.values(): + if not issubclass(provider, PidProvider): + raise TypeError("Argument not an instance of PidProvider.") + pid_type = getattr(provider, 'pid_type', None) + if pid_type is None: + raise AttributeError( + "Provider must specify class variable pid_type.") + pid_type = pid_type.lower() + if pid_type not in registry: + registry[pid_type] = [] + + # Prevent double registration + if provider not in registry[pid_type]: + registry[pid_type].append(provider) + return registry + + registry = LazyDict(__load_providers) + """ Registry of possible providers """ + + pid_type = None + """ + Must be overwritten in subcleass and specified as a string (max len 6) + """ + + @staticmethod + def create(pid_type, pid_str, pid_provider, *args, **kwargs): + """ + Create a new instance of a PidProvider for the + given type and pid. + """ + providers = PidProvider.registry.get(pid_type.lower(), None) + for p in providers: + if p.is_provider_for_pid(pid_str): + return p(*args, **kwargs) + return None + + # + # API methods which must be implemented by each provider. + # + def reserve(self, pid, *args, **kwargs): + """ + Reserve a new persistent identifier + + This might or might not be useful depending on the service of the + provider. + """ + raise NotImplementedError + + def register(self, pid, *args, **kwargs): + """ Register a new persistent identifier """ + raise NotImplementedError + + def update(self, pid, *args, **kwargs): + """ Update information about a persistent identifier """ + raise NotImplementedError + + def delete(self, pid, *args, **kwargs): + """ Delete a persistent identifier """ + raise NotImplementedError + + def sync_status(self, pid, *args, **kwargs): + """ + Synchronize persistent identifier status with remote service provider. + """ + return True + + @classmethod + def is_provider_for_pid(cls, pid_str): + raise NotImplementedError + + # + # API methods which might need to be implemented depending on each provider. + # + def create_new_pid(self, pid_value): + """ Some PidProvider might have the ability to create new values """ + return pid_value + +class LocalPidProvider(PidProvider): + """ + Abstract class for local persistent identifier provides (i.e locally + unmanaged DOIs). + """ + def reserve(self, pid, *args, **kwargs): + pid.log("RESERVE", "Successfully reserved locally") + return True + + def register(self, pid, *args, **kwargs): + pid.log("REGISTER", "Successfully registered in locally") + return True + + def update(self, pid, *args, **kwargs): + # No logging necessary as status of PID is not changing + return True + + def delete(self, pid, *args, **kwargs): + """ Delete a registered DOI """ + pid.log("DELETE", "Successfully deleted locally") + return True diff --git a/modules/miscutil/lib/pid_providers/Makefile.am b/modules/miscutil/lib/pid_providers/Makefile.am new file mode 100644 index 0000000000..551fc6458d --- /dev/null +++ b/modules/miscutil/lib/pid_providers/Makefile.am @@ -0,0 +1,24 @@ +## This file is part of Invenio. +## Copyright (C) 2015 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +pylibdir = $(libdir)/python/invenio/pid_providers + +pylib_DATA = *.py + +EXTRA_DIST = $(pylib_DATA) + +CLEANFILES = *~ *.tmp *.pyc diff --git a/modules/miscutil/lib/pid_providers/__init__.py b/modules/miscutil/lib/pid_providers/__init__.py new file mode 100644 index 0000000000..0eab0f8880 --- /dev/null +++ b/modules/miscutil/lib/pid_providers/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +## +## This file is part of Invenio. +## Copyright (C) 2014 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. diff --git a/modules/miscutil/lib/pid_providers/datacite.py b/modules/miscutil/lib/pid_providers/datacite.py new file mode 100644 index 0000000000..4488c3671b --- /dev/null +++ b/modules/miscutil/lib/pid_providers/datacite.py @@ -0,0 +1,203 @@ +# -*- coding: utf-8 -*- +## +## This file is part of Invenio. +## Copyright (C) 2014, 2015 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +""" + DataCite PID provider. +""" + +from invenio import config +from invenio.dataciteutils import DataCite as DataCiteUtil, HttpError, \ + DataCiteError, DataCiteGoneError, DataCiteNoContentError, \ + DataCiteNotFoundError + +from invenio.pid_provider import PidProvider +from invenio.pid_store import PIDSTORE_STATUS_NEW, \ + PIDSTORE_STATUS_REGISTERED, \ + PIDSTORE_STATUS_DELETED, \ + PIDSTORE_STATUS_RESERVED + + +class DataCite(PidProvider): + """ + DOI provider using DataCite API. + """ + pid_type = 'doi' + + def __init__(self): + self.api = DataCiteUtil() + + def _get_url(self, kwargs): + try: + return kwargs['url'] + except KeyError: + raise Exception("url keyword argument must be specified.") + + def _get_doc(self, kwargs): + try: + return kwargs['doc'] + except KeyError: + raise Exception("doc keyword argument must be specified.") + + def reserve(self, pid, *args, **kwargs): + """ Reserve a DOI (amounts to upload metadata, but not to mint) """ + # Only registered PIDs can be updated. + doc = self._get_doc(kwargs) + + try: + self.api.metadata_post(doc) + except DataCiteError as e: + pid.log("RESERVE", "Failed with %s" % e.__class__.__name__) + return False + except HttpError as e: + pid.log("RESERVE", "Failed with HttpError - %s" % unicode(e)) + return False + else: + pid.log("RESERVE", "Successfully reserved in DataCite") + return True + + def register(self, pid, *args, **kwargs): + """ Register a DOI via the DataCite API """ + url = self._get_url(kwargs) + doc = self._get_doc(kwargs) + + try: + # Set metadata for DOI + self.api.metadata_post(doc) + # Mint DOI + self.api.doi_post(pid.pid_value, url) + except DataCiteError as e: + pid.log("REGISTER", "Failed with %s" % e.__class__.__name__) + return False + except HttpError as e: + pid.log("REGISTER", "Failed with HttpError - %s" % unicode(e)) + return False + else: + pid.log("REGISTER", "Successfully registered in DataCite") + return True + + def update(self, pid, *args, **kwargs): + """ + Update metadata associated with a DOI. + + This can be called before/after a DOI is registered + + """ + url = self._get_url(kwargs) + doc = self._get_doc(kwargs) + + if pid.is_deleted(): + pid.log("UPDATE", "Reactivate in DataCite") + + try: + # Set metadata + self.api.metadata_post(doc) + self.api.doi_post(pid.pid_value, url) + except DataCiteError as e: + pid.log("UPDATE", "Failed with %s" % e.__class__.__name__) + return False + except HttpError as e: + pid.log("UPDATE", "Failed with HttpError - %s" % unicode(e)) + return False + else: + if pid.is_deleted(): + pid.log( + "UPDATE", + "Successfully updated and possibly registered in DataCite" + ) + else: + pid.log("UPDATE", "Successfully updated in DataCite") + return True + + def delete(self, pid, *args, **kwargs): + """ Delete a registered DOI """ + try: + self.api.metadata_delete(pid.pid_value) + except DataCiteError as e: + pid.log("DELETE", "Failed with %s" % e.__class__.__name__) + return False + except HttpError as e: + pid.log("DELETE", "Failed with HttpError - %s" % unicode(e)) + return False + else: + pid.log("DELETE", "Successfully deleted in DataCite") + return True + + def sync_status(self, pid, *args, **kwargs): + """ Synchronize DOI status DataCite MDS """ + status = None + + try: + self.api.doi_get(pid.pid_value) + status = PIDSTORE_STATUS_REGISTERED + except DataCiteGoneError: + status = PIDSTORE_STATUS_DELETED + except DataCiteNoContentError: + status = PIDSTORE_STATUS_REGISTERED + except DataCiteNotFoundError: + pass + except DataCiteError as e: + pid.log("SYNC", "Failed with %s" % e.__class__.__name__) + return False + except HttpError as e: + pid.log("SYNC", "Failed with HttpError - %s" % unicode(e)) + return False + + if status is None: + try: + self.api.metadata_get(pid.pid_value) + status = PIDSTORE_STATUS_RESERVED + except DataCiteGoneError: + status = PIDSTORE_STATUS_DELETED + except DataCiteNoContentError: + status = PIDSTORE_STATUS_REGISTERED + except DataCiteNotFoundError: + pass + except DataCiteError as e: + pid.log("SYNC", "Failed with %s" % e.__class__.__name__) + return False + except HttpError as e: + pid.log("SYNC", "Failed with HttpError - %s" % unicode(e)) + return False + + if status is None: + status = PIDSTORE_STATUS_NEW + + if pid.status != status: + pid.log( + "SYNC", "Fixed status from %s to %s." % (pid.status, status) + ) + pid.status = status + + return True + + @classmethod + def is_provider_for_pid(cls, pid_str): + """ + Check if DataCite is the provider for this DOI + + Note: If you e.g. changed DataCite account and received a new prefix, + then this provider can only update and register DOIs for the new + prefix. + """ + CFG_DATACITE_DOI_PREFIX = getattr(config, + 'CFG_DATACITE_DOI_PREFIX', + '10.0572') + return pid_str.startswith("%s/" % CFG_DATACITE_DOI_PREFIX) + +provider = DataCite diff --git a/modules/miscutil/lib/pid_providers/local_doi.py b/modules/miscutil/lib/pid_providers/local_doi.py new file mode 100644 index 0000000000..adc57490a8 --- /dev/null +++ b/modules/miscutil/lib/pid_providers/local_doi.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +## +## This file is part of Invenio. +## Copyright (C) 2015 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +""" +LocalDOI provider. +""" + +from invenio import config + +from invenio.pid_provider import LocalPidProvider + + +class LocalDOI(LocalPidProvider): + """ + Provider for locally unmanaged DOIs. + """ + pid_type = 'doi' + + @classmethod + def is_provider_for_pid(cls, pid_str): + """ + Check if DOI is not the local datacite managed one. + """ + CFG_DATACITE_DOI_PREFIX = getattr(config, + 'CFG_DATACITE_DOI_PREFIX', + '10.5072') + return pid_str.startswith("%s/" % CFG_DATACITE_DOI_PREFIX) + +provider = LocalDOI diff --git a/modules/miscutil/lib/pid_store.py b/modules/miscutil/lib/pid_store.py new file mode 100644 index 0000000000..13f724ad9f --- /dev/null +++ b/modules/miscutil/lib/pid_store.py @@ -0,0 +1,432 @@ +# -*- coding: utf-8 -*- +## +## This file is part of Invenio. +## Copyright (C) 2015 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +"""PersistentIdentifier store and registration. + +Usage example for registering new identifiers:: + + from flask import url_for + from invenio.pid_store import PersistentIdentifier + + # Reserve a new DOI internally first + pid = PersistentIdentifier.create('doi','10.0572/1234') + + # Get an already reserved DOI + pid = PersistentIdentifier.get('doi', '10.0572/1234') + + # Assign it to a record. + pid.assign('rec', 1234) + + url = url_for("record.metadata", recid=1234, _external=True) + doc = " Date: Wed, 11 Feb 2015 16:34:27 +0100 Subject: [PATCH 14/59] BibFormat: DataCite3 export addition Signed-off-by: Esteban J. G. Gabancho --- .../etc/format_templates/DataCite3.xsl | 232 ++++++++++++++++++ .../etc/format_templates/Makefile.am | 1 + .../bibformat/etc/output_formats/DCITE3.bfo | 1 + .../bibformat/etc/output_formats/Makefile.am | 1 + 4 files changed, 235 insertions(+) create mode 100644 modules/bibformat/etc/format_templates/DataCite3.xsl create mode 100644 modules/bibformat/etc/output_formats/DCITE3.bfo diff --git a/modules/bibformat/etc/format_templates/DataCite3.xsl b/modules/bibformat/etc/format_templates/DataCite3.xsl new file mode 100644 index 0000000000..d76f311a8a --- /dev/null +++ b/modules/bibformat/etc/format_templates/DataCite3.xsl @@ -0,0 +1,232 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <xsl:value-of select="subfield[@code='a']"/> + <xsl:if test="subfield[@code='b']"> + <xsl:text>: </xsl:text><xsl:value-of select="subfield[@code='b']"/> + </xsl:if> + + + + + <xsl:value-of select="subfield[@code='a']"/> + <xsl:if test="subfield[@code='b']"> + <xsl:text>: </xsl:text><xsl:value-of select="subfield[@code='b']"/> + </xsl:if> + + + + + <xsl:value-of select="datafield[@tag=111]/subfield[@code='a']"/> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + [ + + "", + + ] + + + + + diff --git a/modules/bibformat/etc/format_templates/Makefile.am b/modules/bibformat/etc/format_templates/Makefile.am index 5ef4d5b554..3cd74fd1c1 100644 --- a/modules/bibformat/etc/format_templates/Makefile.am +++ b/modules/bibformat/etc/format_templates/Makefile.am @@ -49,6 +49,7 @@ etc_DATA = Default_HTML_captions.bft \ Default_HTML_meta.bft \ WebAuthorProfile_affiliations_helper.bft \ DataCite.xsl \ + DataCite3.xsl \ Default_Mobile_brief.bft \ Default_Mobile_detailed.bft \ Authority_HTML_brief.bft \ diff --git a/modules/bibformat/etc/output_formats/DCITE3.bfo b/modules/bibformat/etc/output_formats/DCITE3.bfo new file mode 100644 index 0000000000..fc1af2db9d --- /dev/null +++ b/modules/bibformat/etc/output_formats/DCITE3.bfo @@ -0,0 +1 @@ +default: DataCite3.xsl diff --git a/modules/bibformat/etc/output_formats/Makefile.am b/modules/bibformat/etc/output_formats/Makefile.am index 2878a3ed0c..b5ecdf9af6 100644 --- a/modules/bibformat/etc/output_formats/Makefile.am +++ b/modules/bibformat/etc/output_formats/Makefile.am @@ -33,6 +33,7 @@ etc_DATA = HB.bfo \ HDFILE.bfo \ XD.bfo \ DCITE.bfo \ + DCITE3.bfo \ WAPAFF.bfo \ WAPDAT.bfo \ XW.bfo \ From 4b8dc9a921f9866a154431c70bb8306f1baf906f Mon Sep 17 00:00:00 2001 From: "Esteban J. G. Gabancho" Date: Thu, 26 Feb 2015 09:05:09 +0100 Subject: [PATCH 15/59] BibField: bug fixes --- modules/bibfield/etc/atlantis.cfg | 16 ++++-- modules/bibfield/lib/bibfield.py | 4 +- .../bibfield/lib/bibfield_config_engine.py | 56 ++++++++++++------- .../lib/bibfield_marcreader_unit_tests.py | 4 +- .../lib/functions/produce_json_for_marc.py | 11 ++-- 5 files changed, 58 insertions(+), 33 deletions(-) diff --git a/modules/bibfield/etc/atlantis.cfg b/modules/bibfield/etc/atlantis.cfg index 8e170656e4..217a13e17a 100644 --- a/modules/bibfield/etc/atlantis.cfg +++ b/modules/bibfield/etc/atlantis.cfg @@ -30,9 +30,9 @@ abstract: ("520__a", "abstract", "summary"), ("520__b", "expansion"), ("520__9", "number")) - marc, "520__", {'summary':value['a'], 'expansion':value['b'], 'number':value['9']} + marc, "520__", {'summary':value['a'], 'expansion':value['b'], 'number':value['9'], 'source': value['2']} producer: - json_for_marc(), {"520__a": "summary", "520__b": "expansion", "520__9": "number"} + json_for_marc(), {"520__a": "summary", "520__b": "expansion", "520__9": "number", "520__2": "source"} json_for_dc(), {"dc:description":"summary"} abstract_french: @@ -123,6 +123,8 @@ aleph_linking_page: json_for_marc(), {"962__a":"type", "962__b":"sysno", "962__l":"library", "962__n":"down_link", "962__m":"up_link", "962__y":"volume_link", "962__p":"part_link", "962__i":"issue_link", "962__k":"pages", "962__t":"base"} authors[0], creator: + schema: + {'authors[0]': {'default': lambda: dict(full_name=None)}} creator: @legacy((("100", "100__", "100__%"), ""), ("100__a", "first author name", "full_name"), @@ -383,7 +385,9 @@ experiment: ('909C0e', 'experiment', '')) marc, "909C0", value['e'] -fft[n]: +fft: + schema: + {'fft': {'type': list, 'force': True}} creator: @legacy(("FFT__a", "path"), ("FFT__d", "description"), @@ -528,7 +532,9 @@ journal_info: json_for_marc(), {"909C4a": "doi","909C4c": "pagination", "909C4d": "date", "909C4e": "recid", "909C4f": "note", "909C4n": "number", "909C4p": "title", "909C4u": "url","909C4v": "volume", "909C4y": "year", "909C4t": "talk", "909C4w": "cnum", "909C4x": "reference"} -keywords[n]: +keywords: + schema: + {'keywords': {'type': list, 'force': True}} creator: @legacy((("653", "6531_", "6531_%"), ""), ("6531_a", "keyword", "term"), @@ -876,7 +882,7 @@ system_number: checker: check_field_existence(0,1) producer: - json_for_marc(), {"970__a": "sysno", "970__d": "recid"} + json_for_marc(), {"970__a": "value", "970__d": "recid"} thesaurus_terms: creator: diff --git a/modules/bibfield/lib/bibfield.py b/modules/bibfield/lib/bibfield.py index 632eb2c498..630613ba89 100644 --- a/modules/bibfield/lib/bibfield.py +++ b/modules/bibfield/lib/bibfield.py @@ -85,7 +85,9 @@ def create_records(blob, master_format='marc', verbose=0, **additional_info): """ record_blods = CFG_BIBFIELD_READERS['bibfield_%sreader.py' % (master_format,)].split_blob(blob, additional_info.get('schema', None)) - return [create_record(record_blob, master_format, verbose=verbose, **additional_info) for record_blob in record_blods] + for record_blob in record_blods: + yield create_record( + record_blob, master_format, verbose=verbose, **additional_info) def get_record(recid, reset_cache=False): diff --git a/modules/bibfield/lib/bibfield_config_engine.py b/modules/bibfield/lib/bibfield_config_engine.py index 3eaf4a2188..4ed2045dd9 100644 --- a/modules/bibfield/lib/bibfield_config_engine.py +++ b/modules/bibfield/lib/bibfield_config_engine.py @@ -147,10 +147,12 @@ def do_unindent(): inherit_from = (Suppress("@inherit_from") + \ originalTextFor(nestedExpr("(", ")")))\ .setResultsName("inherit_from") - override = (Suppress("@") + "override")\ - .setResultsName("override") - extend = (Suppress("@") + "extend")\ - .setResultsName("extend") + override = Suppress("@override") \ + .setResultsName("override") \ + .setParseAction(lambda toks: True) + extend = Suppress("@extend") \ + .setResultsName("extend") \ + .setParseAction(lambda toks: True) master_format = (Suppress("@master_format") + \ originalTextFor(nestedExpr("(", ")")))\ .setResultsName("master_format") \ @@ -298,9 +300,13 @@ def _create(self): to fill up config_rules """ parser = _create_field_parser() - main_rules = parser \ - .parseFile(self.base_dir + '/' + self.main_config_file, - parseAll=True) + try: + main_rules = parser \ + .parseFile(self.base_dir + '/' + self.main_config_file, + parseAll=True) + except ParseException as e: + raise BibFieldParserException( + "Cannot parse file '%s',\n%s" % (self.main_config_file, str(e))) rules = main_rules.rules includes = main_rules.includes already_includes = [self.main_config_file] @@ -310,20 +316,23 @@ def _create(self): if include[0] in already_includes: continue already_includes.append(include[0]) - if os.path.exists(include[0]): - tmp = parser.parseFile(include[0], parseAll=True) - else: - #CHECK: This will raise an IOError if the file doesn't exist - tmp = parser.parseFile(self.base_dir + '/' + include[0], - parseAll=True) + try: + if os.path.exists(include[0]): + tmp = parser.parseFile(include[0], parseAll=True) + else: + #CHECK: This will raise an IOError if the file doesn't exist + tmp = parser.parseFile(self.base_dir + '/' + include[0], + parseAll=True) + except ParseException as e: + raise BibFieldParserException( + "Cannot parse file '%s',\n%s" % (include[0], str(e))) if rules and tmp.rules: rules += tmp.rules else: rules = tmp.rules - if includes and tmp.includes: + + if tmp.includes: includes += tmp.includes - else: - includes = tmp.includes #Create config rules for rule in rules: @@ -378,14 +387,14 @@ def _create_rule(self, rule, override=False, extend=False): % (rule.json_id[0],)) if not json_id in self.__class__._field_definitions and (override or extend): raise BibFieldParserException("Name error: '%s' field name not defined" - % (rule.json_id[0],)) + % (rule.json_id[0],)) #Workaround to keep clean doctype files #Just creates a dict entry with the main json field name and points it to #the full one i.e.: 'authors' : ['authors[0]', 'authors[n]'] - if '[0]' in json_id or '[n]' in json_id: + if ('[0]' in json_id or '[n]' in json_id) and not (extend or override): main_json_id = re.sub('(\[n\]|\[0\])', '', json_id) - if not main_json_id in self.__class__._field_definitions: + if main_json_id not in self.__class__._field_definitions: self.__class__._field_definitions[main_json_id] = [] self.__class__._field_definitions[main_json_id].append(json_id) @@ -526,7 +535,8 @@ def __create_description(self, rule): def __create_producer(self, rule): json_id = rule.json_id[0] - producers = dict() + producers = dict() if not rule.extend else \ + self.__class__._field_definitions[json_id].get('producer', {}) for producer in rule.producer_rule: if producer.producer_code[0][0] not in producers: producers[producer.producer_code[0][0]] = [] @@ -535,8 +545,12 @@ def __create_producer(self, rule): self.__class__._field_definitions[json_id]['producer'] = producers def __create_schema(self, rule): + from invenio.bibfield_utils import CFG_BIBFIELD_FUNCTIONS json_id = rule.json_id[0] - self.__class__._field_definitions[json_id]['schema'] = rule.schema if rule.schema else {} + self.__class__._field_definitions[json_id]['schema'] = \ + try_to_eval(rule.schema.strip(), CFG_BIBFIELD_FUNCTIONS) \ + if rule.schema \ + else self.__class__._field_definitions[json_id].get('schema', {}) def __create_json_extra(self, rule): from invenio.bibfield_utils import CFG_BIBFIELD_FUNCTIONS diff --git a/modules/bibfield/lib/bibfield_marcreader_unit_tests.py b/modules/bibfield/lib/bibfield_marcreader_unit_tests.py index aae14383b2..e690bd9e4b 100644 --- a/modules/bibfield/lib/bibfield_marcreader_unit_tests.py +++ b/modules/bibfield/lib/bibfield_marcreader_unit_tests.py @@ -529,13 +529,13 @@ def test_check_error_reporting(self): reader = MarcReader(blob=xml, schema="xml") r = Record(reader.translate()) - r.check_record(reset = True) + r.check_record(reset=True) self.assertTrue('title' in r) self.assertEquals(len(r['title']), 2) self.assertEquals(len(r.fatal_errors), 1) r['title'] = r['title'][0] - r.check_record(reset = True) + r.check_record(reset=True) self.assertEquals(len(r.fatal_errors), 0) TEST_SUITE = make_test_suite(BibFieldMarcReaderMarcXML, diff --git a/modules/bibfield/lib/functions/produce_json_for_marc.py b/modules/bibfield/lib/functions/produce_json_for_marc.py index f006b4a37a..6d5ec75002 100644 --- a/modules/bibfield/lib/functions/produce_json_for_marc.py +++ b/modules/bibfield/lib/functions/produce_json_for_marc.py @@ -24,6 +24,7 @@ def produce_json_for_marc(self, fields=None): @param tags: list of tags to include in the output, if None or empty list all available tags will be included. """ + from invenio.importutils import try_to_eval from invenio.bibfield_config_engine import get_producer_rules if not fields: fields = self.keys() @@ -50,14 +51,16 @@ def produce_json_for_marc(self, fields=None): tmp_dict[key] = f else: try: - tmp_dict[key] = f[subfield] + tmp_dict[key] = f.get(subfield) except: try: - tmp_dict[key] = self._try_to_eval(subfield, value=f) + tmp_dict[key] = try_to_eval(subfield, self=self, value=f) except Exception as e: - self['__error_messages.cerror[n]'] = 'Producer CError - Unable to produce %s - %s' % (field, str(e)) + self.continuable_errors.append( + 'Producer CError - Unable to produce %s - %s' % (field, str(e))) if tmp_dict: out.append(tmp_dict) except KeyError: - self['__error_messages.cerror[n]'] = 'Producer CError - No producer rule for field %s' % field + self.continuable_errors.append( + 'Producer CError - No producer rule for field %s' % field) return out From 448088f4cf72d36b38bdc02c14c9c18f805067f1 Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Tue, 28 Jun 2011 16:30:20 +0200 Subject: [PATCH 16/59] WebSearch & WebAlert: Your Searches improvements * Moves page "Your Searches" from WebAlert to WebSearch and under the /yoursearches/ URL. Updates all the respective URLs and function calls. Moves function account_list_searches() from WebAlert to WebSearch. * Separates the popular alerts page from the user searches page and creates new separate function for popular alerts the user can choose from to set up new alerts. * Fixes various doc strings, variable and function names. (closes #880) --- .../doc/admin/webalert-admin-guide.webdoc | 2 +- modules/webalert/lib/webalert.py | 136 ++++------- modules/webalert/lib/webalert_templates.py | 227 ++++++++---------- modules/webalert/lib/webalert_webinterface.py | 174 ++++++++------ modules/websearch/lib/Makefile.am | 1 + modules/websearch/lib/websearch_templates.py | 87 +++++++ .../websearch/lib/websearch_webinterface.py | 81 ++++++- .../websearch/lib/websearch_yoursearches.py | 115 +++++++++ .../websession/lib/websession_templates.py | 14 +- .../websession/lib/websession_webinterface.py | 3 +- modules/webstat/etc/webstat.cfg | 4 +- modules/webstat/lib/webstatadmin.py | 4 +- modules/webstyle/lib/webinterface_layout.py | 10 +- 13 files changed, 550 insertions(+), 308 deletions(-) create mode 100644 modules/websearch/lib/websearch_yoursearches.py diff --git a/modules/webalert/doc/admin/webalert-admin-guide.webdoc b/modules/webalert/doc/admin/webalert-admin-guide.webdoc index 570843348b..5ba2a2d305 100644 --- a/modules/webalert/doc/admin/webalert-admin-guide.webdoc +++ b/modules/webalert/doc/admin/webalert-admin-guide.webdoc @@ -53,7 +53,7 @@ module to permit this functionality.

Configuring Alert Queries

Users may set up alert queries for example from their search history pages. +href="/yoursearches/display">search history pages.

Administrators may edit existing users' alerts by modifying the user_query_basket table. (There is no web interface yet diff --git a/modules/webalert/lib/webalert.py b/modules/webalert/lib/webalert.py index 8fd46db718..664e9889e1 100644 --- a/modules/webalert/lib/webalert.py +++ b/modules/webalert/lib/webalert.py @@ -54,8 +54,10 @@ def check_alert_name(alert_name, uid, ln=CFG_SITE_LANG): raise AlertError( _("You already have an alert named %s.") % ('' + cgi.escape(alert_name) + '',) ) def get_textual_query_info_from_urlargs(urlargs, ln=CFG_SITE_LANG): - """Return nicely formatted search pattern and catalogue from urlargs of the search query. - Suitable for 'your searches' display.""" + """ + Return nicely formatted search pattern and catalogue from urlargs of the search query. + """ + out = "" args = cgi.parse_qs(urlargs) return webalert_templates.tmpl_textual_query_info_from_urlargs( @@ -64,71 +66,6 @@ def get_textual_query_info_from_urlargs(urlargs, ln=CFG_SITE_LANG): ) return out -def perform_display(permanent, uid, ln=CFG_SITE_LANG): - """display the searches performed by the current user - input: default permanent="n"; permanent="y" display permanent queries(most popular) - output: list of searches in formatted html - """ - # load the right language - _ = gettext_set_language(ln) - - # first detect number of queries: - nb_queries_total = 0 - nb_queries_distinct = 0 - query = "SELECT COUNT(*),COUNT(DISTINCT(id_query)) FROM user_query WHERE id_user=%s" - res = run_sql(query, (uid,), 1) - try: - nb_queries_total = res[0][0] - nb_queries_distinct = res[0][1] - except: - pass - - # query for queries: - params = () - if permanent == "n": - SQL_query = "SELECT DISTINCT(q.id),q.urlargs "\ - "FROM query q, user_query uq "\ - "WHERE uq.id_user=%s "\ - "AND uq.id_query=q.id "\ - "ORDER BY q.id DESC" - params = (uid,) - else: - # permanent="y" - SQL_query = "SELECT q.id,q.urlargs "\ - "FROM query q "\ - "WHERE q.type='p'" - query_result = run_sql(SQL_query, params) - - queries = [] - if len(query_result) > 0: - for row in query_result : - if permanent == "n": - res = run_sql("SELECT DATE_FORMAT(MAX(date),'%%Y-%%m-%%d %%H:%%i:%%s') FROM user_query WHERE id_user=%s and id_query=%s", - (uid, row[0])) - try: - lastrun = res[0][0] - except: - lastrun = _("unknown") - else: - lastrun = "" - queries.append({ - 'id' : row[0], - 'args' : row[1], - 'textargs' : get_textual_query_info_from_urlargs(row[1], ln=ln), - 'lastrun' : lastrun, - }) - - - return webalert_templates.tmpl_display_alerts( - ln = ln, - permanent = permanent, - nb_queries_total = nb_queries_total, - nb_queries_distinct = nb_queries_distinct, - queries = queries, - guest = isGuestUser(uid), - guesttxt = warning_guest_user(type="alerts", ln=ln) - ) - def check_user_can_add_alert(id_user, id_query): """Check if ID_USER has really alert adding rights on ID_QUERY (that is, the user made the query herself or the query is one of @@ -236,11 +173,20 @@ def perform_add_alert(alert_name, frequency, notification, run_sql(query, params) out = _("The alert %s has been added to your profile.") out %= '' + cgi.escape(alert_name) + '' - out += perform_list_alerts(uid, ln=ln) + out += perform_request_youralerts_display(uid, ln=ln) return out -def perform_list_alerts(uid, ln=CFG_SITE_LANG): - """perform_list_alerts display the list of alerts for the connected user""" +def perform_request_youralerts_display(uid, + ln=CFG_SITE_LANG): + """ + Display a list of the user defined alerts. + @param uid: The user id + @type uid: int + @param ln: The interface language + @type ln: string + @return: HTML formatted list of the user defined alerts. + """ + # set variables out = "" @@ -284,9 +230,10 @@ def perform_list_alerts(uid, ln=CFG_SITE_LANG): register_exception(alert_admin=True) # link to the "add new alert" form - out = webalert_templates.tmpl_list_alerts(ln=ln, alerts=alerts, - guest=isGuestUser(uid), - guesttxt=warning_guest_user(type="alerts", ln=ln)) + out = webalert_templates.tmpl_youralerts_display(ln=ln, + alerts=alerts, + guest=isGuestUser(uid), + guesttxt=warning_guest_user(type="alerts", ln=ln)) return out def perform_remove_alert(alert_name, id_query, id_basket, uid, ln=CFG_SITE_LANG): @@ -314,7 +261,7 @@ def perform_remove_alert(alert_name, id_query, id_basket, uid, ln=CFG_SITE_LANG) out += "The alert %s has been removed from your profile.

\n" % cgi.escape(alert_name) else: out += "Unable to remove alert %s.

\n" % cgi.escape(alert_name) - out += perform_list_alerts(uid, ln=ln) + out += perform_request_youralerts_display(uid, ln=ln) return out @@ -374,7 +321,7 @@ def perform_update_alert(alert_name, frequency, notification, id_basket, id_quer run_sql(query, params) out += _("The alert %s has been successfully updated.") % ("" + cgi.escape(alert_name) + "",) - out += "

\n" + perform_list_alerts(uid, ln=ln) + out += "

\n" + perform_request_youralerts_display(uid, ln=ln) return out def is_selected(var, fld): @@ -409,21 +356,32 @@ def account_list_alerts(uid, ln=CFG_SITE_LANG): return webalert_templates.tmpl_account_list_alerts(ln=ln, alerts=alerts) -def account_list_searches(uid, ln=CFG_SITE_LANG): - """ account_list_searches: list the searches of the user - input: the user id - output: resume of the searches""" - out = "" - # first detect number of queries: - nb_queries_total = 0 - res = run_sql("SELECT COUNT(*) FROM user_query WHERE id_user=%s", (uid,), 1) - try: - nb_queries_total = res[0][0] - except: - pass +def perform_request_youralerts_popular(ln=CFG_SITE_LANG): + """ + Display popular alerts. + @param uid: the user id + @type uid: integer + @return: A list of searches queries in formatted html. + """ # load the right language _ = gettext_set_language(ln) - out += _("You have made %(x_nb)s queries. A %(x_url_open)sdetailed list%(x_url_close)s is available with a possibility to (a) view search results and (b) subscribe to an automatic email alerting service for these queries.") % {'x_nb': nb_queries_total, 'x_url_open': '' % ln, 'x_url_close': ''} - return out + # fetch the popular queries + query = """ SELECT q.id, + q.urlargs + FROM query q + WHERE q.type='p'""" + result = run_sql(query) + + search_queries = [] + if result: + for search_query in result: + search_query_id = search_query[0] + search_query_args = search_query[1] + search_queries.append({'id' : search_query_id, + 'args' : search_query_args, + 'textargs' : get_textual_query_info_from_urlargs(search_query_args, ln=ln)}) + + return webalert_templates.tmpl_youralerts_popular(ln = ln, + search_queries = search_queries) diff --git a/modules/webalert/lib/webalert_templates.py b/modules/webalert/lib/webalert_templates.py index 80e75bf05b..782b94d163 100644 --- a/modules/webalert/lib/webalert_templates.py +++ b/modules/webalert/lib/webalert_templates.py @@ -107,7 +107,7 @@ def tmpl_account_list_alerts(self, ln, alerts): # load the right message language _ = gettext_set_language(ln) - out = """

+ out = """ %(you_own)s: + +
+ """ % {'search_text': _('Search inside your searches for'), + 'action': '%s/yoursearches/display?ln=%s' % (CFG_SITE_SECURE_URL, ln), + 'p': cgi.escape(p), + 'submit_label': _('Search')} + out += '

' + search_form + '

' + + counter = (page - 1) * step + yoursearches = "" + for search_query in search_queries: + counter += 1 + search_query_args = search_query['args'] + search_query_id = search_query['id'] + search_query_lastrun = search_query['lastrun'] + search_query_number_of_user_alerts = search_query['user_alerts'] + + search_query_details = get_html_user_friendly_search_query_args(search_query_args, ln) + + search_query_last_performed = get_html_user_friendly_search_query_lastrun(search_query_lastrun, ln) + + search_query_options_search = """%s""" % \ + (CFG_SITE_SECURE_URL, cgi.escape(search_query_args), CFG_SITE_URL, _('Search again')) + + search_query_options_alert = search_query_number_of_user_alerts and \ + """%s""" % \ + (CFG_SITE_SECURE_URL, ln, search_query_id, CFG_SITE_URL, _('Edit your existing alert(s)')) + \ + '   ' + \ + """%s""" % \ + (CFG_SITE_SECURE_URL, ln, search_query_id, CFG_SITE_URL, _('Set up a new alert')) or \ + """%s""" % \ + (CFG_SITE_SECURE_URL, ln, search_query_id, CFG_SITE_URL, _('Set up a new alert')) + + search_query_options = "%s   %s" % \ + (search_query_options_search, \ + search_query_options_alert) + + yoursearches += """ + + + %(counter)i. + + +
%(search_query_details)s
+
%(search_query_last_performed)s
+
%(search_query_options)s
+ + """ % {'counter': counter, + 'search_query_details': search_query_details, + 'search_query_last_performed': search_query_last_performed, + 'search_query_options': search_query_options} + + footer = '' + if paging_navigation[0]: + footer += """""" % \ + (CFG_SITE_SECURE_URL, 1, step, cgi.escape(p), ln, '/img/yoursearches_first_page.png') + if paging_navigation[1]: + footer += """""" % \ + (CFG_SITE_SECURE_URL, page - 1, step, cgi.escape(p), ln, '/img/yoursearches_previous_page.png') + footer += " " + displayed_searches_from = ((page - 1) * step) + 1 + displayed_searches_to = paging_navigation[2] and (page * step) or nb_queries_distinct + footer += _('Displaying searches %i to %i from %i total unique searches') % \ + (displayed_searches_from, displayed_searches_to, nb_queries_distinct) + footer += " " + if paging_navigation[2]: + footer += """""" % \ + (CFG_SITE_SECURE_URL, page + 1, step, cgi.escape(p), ln, '/img/yoursearches_next_page.png') + if paging_navigation[3]: + footer += """""" % \ + (CFG_SITE_SECURE_URL, paging_navigation[3], step, cgi.escape(p), ln, '/img/yoursearches_last_page.png') + + out += """ + + + + + + + + + + + + + %(yoursearches)s + +
%(footer)s
""" % {'header': _('Search details'), + 'footer': footer, + 'yoursearches': yoursearches} return out + +def get_html_user_friendly_search_query_args(args, + ln=CFG_SITE_LANG): + """ + Internal function. + Returns an HTML formatted user friendly description of a search query's + arguments. + + @param args: The search query arguments as they apear in the search URL + @type args: string + + @param ln: The language to display the interface in + @type ln: string + + @return: HTML formatted user friendly description of a search query's + arguments + """ + + # Load the right language + _ = gettext_set_language(ln) + + # Arguments dictionary + dict = parse_qs(args) + + if not dict.has_key('p') and not dict.has_key('p1') and not dict.has_key('p2') and not dict.has_key('p3'): + search_patterns_html = _('Search for everything') + else: + search_patterns_html = _('Search for') + ' ' + if dict.has_key('p'): + search_patterns_html += '' + cgi.escape(dict['p'][0]) + '' + if dict.has_key('f'): + search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(dict['f'][0]) + '' + if dict.has_key('p1'): + if dict.has_key('p'): + search_patterns_html += ' ' + _('and') + ' ' + search_patterns_html += '' + cgi.escape(dict['p1'][0]) + '' + if dict.has_key('f1'): + search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(dict['f1'][0]) + '' + if dict.has_key('p2'): + if dict.has_key('p') or dict.has_key('p1'): + if dict.has_key('op1'): + search_patterns_html += ' %s ' % (dict['op1'][0] == 'a' and _('and') or \ + dict['op1'][0] == 'o' and _('or') or \ + dict['op1'][0] == 'n' and _('and not') or + ', ',) + search_patterns_html += '' + cgi.escape(dict['p2'][0]) + '' + if dict.has_key('f2'): + search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(dict['f2'][0]) + '' + if dict.has_key('p3'): + if dict.has_key('p') or dict.has_key('p1') or dict.has_key('p2'): + if dict.has_key('op2'): + search_patterns_html += ' %s ' % (dict['op2'][0] == 'a' and _('and') or \ + dict['op2'][0] == 'o' and _('or') or \ + dict['op2'][0] == 'n' and _('and not') or + ', ',) + search_patterns_html += '' + cgi.escape(dict['p3'][0]) + '' + if dict.has_key('f3'): + search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(dict['f3'][0]) + '' + + if not dict.has_key('c') and not dict.has_key('cc'): + collections_html = _('in all the collections') + else: + collections_html = _('in the following collection(s)') + ': ' + if dict.has_key('c'): + collections_html += ', '.join('' + cgi.escape(collection) + '' for collection in dict['c']) + elif dict.has_key('cc'): + collections_html += '' + cgi.escape(dict['cc'][0]) + '' + + search_query_args_html = search_patterns_html + '
' + collections_html + + return search_query_args_html + + +def get_html_user_friendly_search_query_lastrun(lastrun, + ln=CFG_SITE_LANG): + """ + Internal function. + Returns an HTML formatted user friendly description of a search query's + last run date. + + @param lastrun: The search query last run date in the following format: + '2005-11-16 15:11:57' + @type lastrun: string + + @param ln: The language to display the interface in + @type ln: string + + @return: HTML formatted user friendly description of a search query's + last run date + """ + + # Load the right language + _ = gettext_set_language(ln) + + # Calculate how many days old the search query is base on the lastrun date + # and today + lastrun_datestruct = convert_datetext_to_datestruct(lastrun) + today = datetime_date.today() + if lastrun_datestruct.tm_year != 0 and \ + lastrun_datestruct.tm_mon != 0 and \ + lastrun_datestruct.tm_mday != 0: + days_old = (today - datetime_date(lastrun_datestruct.tm_year, + lastrun_datestruct.tm_mon, + lastrun_datestruct.tm_mday)).days + if days_old == 0: + out = _('Today') + elif days_old < 7: + out = str(days_old) + ' ' + _('day(s) ago') + elif days_old == 7: + out = _('A week ago') + elif days_old < 14: + out = _('More than a week ago') + elif days_old == 14: + out = _('Two weeks ago') + elif days_old < 30: + out = _('More than two weeks ago') + elif days_old == 30: + out = _('A month ago') + elif days_old < 180: + out = _('More than a month ago') + elif days_old < 365: + out = _('More than six months ago') + else: + out = _('More than a year ago') + out += '' + \ + ' ' + _('on') + ' ' + lastrun.split()[0] + \ + ' ' + _('at') + ' ' + lastrun.split()[1] + \ + '' + else: + out = _('Unknown') + + return out diff --git a/modules/websearch/lib/websearch_webinterface.py b/modules/websearch/lib/websearch_webinterface.py index 9c2bfdee12..9f779e46bc 100644 --- a/modules/websearch/lib/websearch_webinterface.py +++ b/modules/websearch/lib/websearch_webinterface.py @@ -1212,14 +1212,17 @@ class WebInterfaceYourSearchesPages(WebInterfaceDirectory): def index(self, req, form): """ """ - redirect_to_url(req, '%s/yoursearches/display' % CFG_SITE_URL) + redirect_to_url(req, '%s/yoursearches/display' % CFG_SITE_SECURE_URL) def display(self, req, form): """ Display the user's search latest history. """ - argd = wash_urlargd(form, {'ln': (str, "en")}) + argd = wash_urlargd(form, {'ln': (str, 'en'), + 'page': (int, 1), + 'step': (int, 20), + 'p': (str, '')}) uid = getUid(req) @@ -1228,14 +1231,14 @@ def display(self, req, form): if CFG_ACCESS_CONTROL_LEVEL_SITE >= 1: return page_not_authorized(req, "%s/yoursearches/display" % \ - (CFG_SITE_URL,), + (CFG_SITE_SECURE_URL,), navmenuid="yoursearches") elif uid == -1 or isGuestUser(uid): return redirect_to_url(req, "%s/youraccount/login%s" % \ (CFG_SITE_SECURE_URL, make_canonical_urlargd( {'referer' : "%s/yoursearches/display%s" % ( - CFG_SITE_URL, + CFG_SITE_SECURE_URL, make_canonical_urlargd(argd, {})), 'ln' : argd['ln']}, {}) @@ -1259,7 +1262,11 @@ def display(self, req, form): suffix="Do the webstat tables exists? Try with 'webstatadmin --load-config'") return page(title=_("Your Searches"), - body=perform_request_yoursearches_display(uid, ln=argd['ln']), + body=perform_request_yoursearches_display(uid, + page=argd['page'], + step=argd['step'], + p=argd['p'], + ln=argd['ln']), navtrail= """%(account)s""" % { 'sitesecureurl' : CFG_SITE_SECURE_URL, 'ln': argd['ln'], diff --git a/modules/websearch/lib/websearch_yoursearches.py b/modules/websearch/lib/websearch_yoursearches.py index 47ee1dda11..6df63ad3d3 100644 --- a/modules/websearch/lib/websearch_yoursearches.py +++ b/modules/websearch/lib/websearch_yoursearches.py @@ -19,51 +19,112 @@ __revision__ = "$Id$" -from invenio.config import CFG_SITE_LANG, CFG_SITE_URL +from invenio.config import CFG_SITE_LANG, CFG_SITE_SECURE_URL from invenio.dbquery import run_sql from invenio.webaccount import warning_guest_user -from invenio.webalert import get_textual_query_info_from_urlargs from invenio.messages import gettext_set_language from invenio.webuser import isGuestUser +from urllib import quote, quote_plus, unquote_plus +from invenio.webalert import count_user_alerts_for_given_query import invenio.template websearch_templates = invenio.template.load('websearch') +CFG_WEBSEARCH_YOURSEARCHES_MAX_NUMBER_OF_DISPLAYED_SEARCHES = 20 + def perform_request_yoursearches_display(uid, + page=1, + step=CFG_WEBSEARCH_YOURSEARCHES_MAX_NUMBER_OF_DISPLAYED_SEARCHES, + p='', ln=CFG_SITE_LANG): """ Display the user's search history. + @param uid: the user id @type uid: integer + + @param page: + @type page: integer + + @param step: + @type step: integer + + @param p: + @type p: strgin + + @param ln: + @type ln: string + @return: A list of searches queries in formatted html. """ - - # load the right language + # Load the right language _ = gettext_set_language(ln) - # firstly, calculate the number of total and distinct queries + search_clause = "" + if p: + p_stripped = p.strip() + p_stripped_args = p.split() + sql_p_stripped_args = ['\'%%' + quote(p_stripped_arg) + '%%\'' for p_stripped_arg in p_stripped_args] + for sql_p_stripped_arg in sql_p_stripped_args: + search_clause += """ AND q.urlargs LIKE %s""" % (sql_p_stripped_arg,) + + # Calculate the number of total and distinct queries nb_queries_total = 0 nb_queries_distinct = 0 - query_nb_queries = """ SELECT COUNT(*), - COUNT(DISTINCT(id_query)) - FROM user_query - WHERE id_user=%s""" + query_nb_queries = """ SELECT COUNT(uq.id_query), + COUNT(DISTINCT(uq.id_query)) + FROM user_query AS uq, + query q + WHERE uq.id_user=%%s + AND q.id=uq.id_query + %s""" % (search_clause,) params_nb_queries = (uid,) res_nb_queries = run_sql(query_nb_queries, params_nb_queries) nb_queries_total = res_nb_queries[0][0] nb_queries_distinct = res_nb_queries[0][1] - # secondly, calculate the search queries + # The real page starts counting from 0, i.e. minus 1 from the human page + real_page = page - 1 + # The step needs to be a positive integer + if (step <= 0): + step = CFG_WEBSEARCH_YOURSEARCHES_MAX_NUMBER_OF_DISPLAYED_SEARCHES + # The maximum real page is the integer division of the total number of + # searches and the searches displayed per page + max_real_page = (nb_queries_distinct / step) - (not (nb_queries_distinct % step) and 1 or 0) + # Check if the selected real page exceeds the maximum real page and reset + # if needed + if (real_page >= max_real_page): + #if ((nb_queries_distinct % step) != 0): + # real_page = max_real_page + #else: + # real_page = max_real_page - 1 + real_page = max_real_page + page = real_page + 1 + elif (real_page < 0): + real_page = 0 + page = 1 + # Calculate the start value for the SQL LIMIT constraint + limit_start = real_page * step + # Calculate the display of the paging navigation arrows for the template + paging_navigation = (real_page >= 2, + real_page >= 1, + real_page <= (max_real_page - 1), + (real_page <= (max_real_page - 2)) and (max_real_page + 1)) + + + # Calculate the user search queries query = """ SELECT DISTINCT(q.id), q.urlargs, - DATE_FORMAT(MAX(uq.date),'%%Y-%%m-%%d %%H:%%i:%%s') + DATE_FORMAT(MAX(uq.date),'%s') FROM query q, user_query uq - WHERE uq.id_user=%s + WHERE uq.id_user=%%s AND uq.id_query=q.id + %s GROUP BY uq.id_query - ORDER BY q.id DESC""" - params = (uid,) + ORDER BY MAX(uq.date) DESC + LIMIT %%s,%%s""" % ('%%Y-%%m-%%d %%H:%%i:%%s', search_clause,) + params = (uid, limit_start, step) result = run_sql(query, params) search_queries = [] @@ -74,16 +135,20 @@ def perform_request_yoursearches_display(uid, search_query_lastrun = search_query[2] or _("unknown") search_queries.append({'id' : search_query_id, 'args' : search_query_args, - 'textargs' : get_textual_query_info_from_urlargs(search_query_args, ln=ln), - 'lastrun' : search_query_lastrun}) + 'lastrun' : search_query_lastrun, + 'user_alerts' : count_user_alerts_for_given_query(uid, search_query_id)}) return websearch_templates.tmpl_yoursearches_display( - ln = ln, nb_queries_total = nb_queries_total, nb_queries_distinct = nb_queries_distinct, search_queries = search_queries, + page=page, + step=step, + paging_navigation=paging_navigation, + p=p, guest = isGuestUser(uid), - guesttxt = warning_guest_user(type="searches", ln=ln)) + guesttxt = warning_guest_user(type="searches", ln=ln), + ln = ln) def account_list_searches(uid, ln=CFG_SITE_LANG): @@ -109,7 +174,7 @@ def account_list_searches(uid, out = _("You have made %(x_nb)s queries. A %(x_url_open)sdetailed list%(x_url_close)s is available with a possibility to (a) view search results and (b) subscribe to an automatic email alerting service for these queries.") % \ {'x_nb': nb_queries_total, - 'x_url_open': '' % (CFG_SITE_URL, ln), + 'x_url_open': '' % (CFG_SITE_SECURE_URL, ln), 'x_url_close': ''} - return out \ No newline at end of file + return out diff --git a/modules/webstyle/css/invenio.css b/modules/webstyle/css/invenio.css index 288e75eaa2..b8753925a5 100644 --- a/modules/webstyle/css/invenio.css +++ b/modules/webstyle/css/invenio.css @@ -2568,6 +2568,7 @@ a:hover.bibMergeImgClickable img { background:Yellow; } /* end of BibMerge module */ + /* WebAlert module */ .alrtTable{ border: 1px solid black; @@ -2581,6 +2582,93 @@ a:hover.bibMergeImgClickable img { } /* end of WebAlert module */ +/* WebSearch module - YourSearches */ +.websearch_yoursearches_table { + border: 1px solid #bbbbbb; +} +.websearch_yoursearches_table_header td { +/* background-color: #63a5cd;*/ + background-color: #7a9bdd; + color: white; + padding: 2px; + font-weight: bold; + font-size: 75%; + text-transform: uppercase; +/* border-bottom: 1px solid #327aa5; + border-right: 1px solid #327aa5;*/ + border-bottom: 1px solid #3366cc; + border-right: 1px solid #3366cc; + white-space: nowrap; + vertical-align: top; +} +.websearch_yoursearches_table_footer td { +/* background-color: #63a5cd;*/ + background-color: #7a9bdd; + color: white; + padding: 5px; + font-size: 80%; + text-align: center; +/* border-bottom: 1px solid #327aa5; + border-right: 1px solid #327aa5;*/ + border-bottom: 1px solid #3366cc; + border-right: 1px solid #3366cc; + white-space: nowrap; +} +.websearch_yoursearches_table_footer img { + border: none; + margin: 0px 3px; + vertical-align: bottom; +} +.websearch_yoursearches_footer a, .websearch_yoursearches_footer a:link, .websearch_yoursearches_footer a:visited, .websearch_yoursearches_footer a:active { + text-decoration: none; + color: #000; +} +.websearch_yoursearches_footer a:hover { + text-decoration: underline; + color: #000; +} +.websearch_yoursearches_footer img { + border: none; + vertical-align: bottom; +} +.websearch_yoursearches_table_content { + background-color: #f2f2fa; + padding: 5px; + text-align: left; + vertical-align: top; +} +.websearch_yoursearches_table_content_mouseover { + background-color: #ebebfa; + padding: 5px; + text-align: left; + vertical-align: top; +} +.websearch_yoursearches_table_counter { + background-color: #f2f2fa; + padding: 5px 2px; + text-align: right; + vertical-align: top; + font-size: 80%; + color: grey; +} +.websearch_yoursearches_table_content_options { + margin: 5px; + font-size: 80%; +} +.websearch_yoursearches_table_content_options img{ + vertical-align: middle; + margin-right: 3px; +} +.websearch_yoursearches_table_content_options a, .websearch_yoursearches_table_content_options a:link, .websearch_yoursearches_table_content_options a:visited, .websearch_yoursearches_table_content_options a:active { + text-decoration: underline; + color: #333; +} +.websearch_yoursearches_table_content_options a:hover { + text-decoration: underline; + color: #000; +} +/* end of WebSearches module - YourSearches */ + /* BibClassify module */ .bibclassify { text-align: center; diff --git a/modules/webstyle/img/yoursearches_alert.png b/modules/webstyle/img/yoursearches_alert.png new file mode 100644 index 0000000000000000000000000000000000000000..c036ff6de4ff10cfb3b89b0c52740898b78639ee GIT binary patch literal 991 zcmV<510ei~P)w5Rx83Uv#ml(IHCMomsEWAF^>~ z?zaaWadt7U(R1e^AIE<%`o;KfUq0*R`n8>YAge7Iv+Z=3}v#78e&) z3JVJhipS#`85v0xPb=y0@KCv2PN7gph0@l>D^m1D{r&?wO|UGhB>NL}m!(ZkABV|~ z!TIR3M)UYqgk&=LNK1E@o|iU`75cvaSXLvS&nutL<8(S7 zop{@Te@@;X2crq2Y<_;;E+x_I>>bCQJ~*)pCBa+8m*@KMc`v&8ER>l*6Xc@M{}Wkb zdU`tB(9lp84u|RJ=%BH&5!<#o&zFbca5&iB-e!4unUSLLBqT|@ySutjSE10?WlH|y)>uEPsjf)B`r5Vn zk_8IHnM_8pSgh#V!^OvMPwBG{Ov(OR+VY;RgS~;SzQE}S=2;0+sT2nX2e@1=d_Erl zhEeQm;rz|FWAwZsxLXc+3yeY>hCc=%1Yk-s5{U%aY!;8lLv?j^N#e7(cmDx(?w|2q z5T#b3;S#+67B2ypjDdlH3ez;}|I5aBJf8eVxB74OUEhB9y(^$YFtR`%coL`wemlM@ zC`VuXh+**m@Z9A_Zulzbto3m5VlBpFE;!7#=V1(qz|5JoNt2=^DCe9T zbFQSJHC^RGnfPB@vp@Qy%|Bdi*jk1g%jJ;li`}`{a8vXgGo7f1fWHQGf_kE!-W>-V z-mmXH{RLAa*Xz&E_PjsO`~5tR+P{Cl+itg40$>dCS zX0y@S`rD*33?ornTdRA$UQE-(W=o^<sM&%(4j*K zo6W{F@pzogn>Sk#5$Wpc5)qN^^W7pMa<>1bkV>Uwn!%v~IdS@&0O&54%bb=OdR(Zz zl0KTccI}#A7)HYH_e)1dhdhhVcSqXW|CEOhA4+p`^ThfOKDX)sd3kwMS635_MzLD0 zw70iYT3X6vWP;MtH*h!{*z=3C)@`uw!Z3_PFc_5CXgl5~pYAy+SFZMp&*u}KL)wSm zbjZ4|jtY*3Q>P|XQ&UsY;c$q^>;jsm;rIJBP1CIP_4O%4M2;LeBEeu#o{Y>i zO&J*(7N^rGA|g#c@5|Y+ahru?GAXH4N}|!Iw6wI$6tuRs%7u&F(to8-L`04rKRed$ zz9=f0Op3)~K|}}y0))e1B2y7GO(P>cgHR|)L18iOzb<$N2Zt+W3MM8d$jZt>DFwj2 z`=cZiNutpt1KwNQxq02bqhV)AnWibGX=1fn|7CdVcc&QjjdEbuYR+Fcw{K)*c&BnY zo#wLJ7!I??Bm9RmD_$1d+C1#flml z8bScNrfF1ER8Un_g@`abJWOY2C(D;Fr@XxUDY5BKP1EELXB$ROJmuquig@bI9<#HON2blobE zsc`Yim6gZF!{ccdT~qu|EWrGa!ybQbuQILcfP+W1LkXEA-gw{}sh(r-lt5%H27Em;jAh-~gYH7iUG*r6pLpz=2 zef}twLWt+r&G~Zg{c_IvKE%Yv!~+25JO%*x)T!>o*7g>9Y{}w1z1|)|Qc~hQxDHE< z#mOB``++`hFSWL{XftPel8~9{n&ZyNx{{OQ&S~#x)v~g)5;8JeDV^ONl$w?@m*4-u zyg;@)gHD_{dBQ&!V5Oy{gTMWD>RfkMx3+N6!+toNafP@6{fM&;E$vMN0NZzLr}mCk z0)S0r8>y|mg;<}rmjb~67-Qg!BNz;V>o^z~_@I9fTQ*m+k3QV#A<{5PiDuQGeK8pz zATe>mea$WZP_{es%837%JIDQ$&o@9eyKMphsHzG8NJ>sx+}zwm-a8&R4yh$$z5JdwdNk(2?9=%>#maucz`Q)YQ}v5fK1BT!?en<4Q```KqcurI8mDB`W&Gmsa4yBPsi~plw!y`{u&l%*a~dlvGwDCF1|V0K`slrG8oZYKb=?;m&zHOp-LGGagY&BpeP0G))TufXCyx>7&d{*Q2YR zUU})l#rj}Rcju+Od%w9WD>EDM&iK&n8cRz}xwmNHLl@4TJrnBb?P@-9NpR zZEY>)7cMk5G+rzzDq6U#sHiA5Dk=(ozn=gwB7p0300^=yYl1Ax8i*hoi~s%Cv+Ra92|T66Aq#9FLO6h$%`Ob9bcFw5bH2#-WWcoYOd zfNCAB33j`UL{UT_7=S1mV6~3cw!O3M7=ZD`Yt~`?hBqe~jpEBrNBok)2j=72fR~_w zan2!065?!jzWVvqKLN-hB5W$#7_U)qx!qgwvt z3l=O`OGIl={``}4?%e5IizT)qH+RN3lhFjLY9)5Q?vK&6=l@6zX(1vaL`Fsq%(}ba zIAct^HIlWpwU{(%(tmsP@OKAYDM{%UR8{S{-K-GN@a%$!@ZI-^Cy!Drt^WevHBK#s Snf#mp0000 literal 0 HcmV?d00001 diff --git a/modules/webstyle/img/yoursearches_last_page.png b/modules/webstyle/img/yoursearches_last_page.png new file mode 100644 index 0000000000000000000000000000000000000000..c61494c785551a39a17c19ea3dcd458e53aeec0a GIT binary patch literal 1548 zcmV+n2J`ueP)-nVx{-B{UR182g5kqCO93Cac<2TJ&u7!`E5P5BbWq7Eo!>pIwM*WT`nj^5te z-uB+U#~&M7NsQ00lPAwP?>WyoFaBu$Z48D`TX+0X*z84%13iAxKhQ^yJwDF_0LZdz z1%Q@T=NR%}_wOaR;l$4C9-v31~eR%qjarfpm z+iXRx$S=76$xD|m3IHf5D45vhb`b!~nmw~?+VrVUfQW820V3kJ;zKnDilAvKeRW_T z0YD_8(a~>e2mn+qOsuJxY2^CnX#rAq2g@>Aj}y72C(^u_0bnNF)-pW$PyQ;wS9BSWKrERk=hYTV zRF6Q{qmU#CAML2NmI(L**S%YHiF`0KUY+} zci#;08V%gMA)HsQQ0Z%>bBTzqUUiaWnkGWZO@IMH2=>6#DTruKH)xS4lyC@)bC3{h z%-y50V#RZH0AQLXxDWy0)@}&DZVjb~hQOL569Nu187JRH5q;>h0^LJ)c!}t}A_cLw}Ew z3ILUrmE(T+@faDVL8fVv9@GEU2M2<5@bCfom33>kIW9ZXhzQG9Eb|f(nF*6xT`jbJ z!>cw)mIK^03}~7`02c7mM+~15#M?0^hve4`iqM>ql*!VXkd(i zF$N;R;-&yWVE+8a?_RKA;YQB+#)}u5btM#%03aHTz=)gh`(-W#0^Q6sO&Dez7A}Ev z27rNqfip(d%#rNOnNz6w`pBQHEp6*Yjmo~Js_ONB|78ZrB7vBo5gD1x?QTQWp4~y` zm6p|!s9FUeR|o+jf}(_(q$tYZ-Utws0LX$&YbLgrZ^MPg3!izt-cM4bbjSI|Qv?7r zADTH4U@%Q%$Q97YtZY`bw+iRZo;`8>ddH@$5xEzR|9Dj7jvCMMCS(Z!(Bn}s280!b zNoqLErIZvTOo8%>a;3KR#A~wOegpN~lv3)0huW6RdfvHJR- zH~M^C-*tI?zQNZ2f!yED#*Qx>oGBpyFsQ1^S$0l#UV7SHV>&xMzXM3%f517<89(8^ yNo_87BLL%ngg7 z&2i^-$G*jWuh&~5+=Fh#Xj}@Lt>sZ7`IU7&$LHG4_*_q22I-yjD&M}RAxOYrFyQ8` zo6-IkuXZYujHsyCymyDC+6h2l5mL(ac=p^?1>m>706_l0G`oif1BJ^Ax7XHK;|t2W zskfX+er0CniPkiC#!;LBcWg+9#=@V%HYw-uv`>?yFpkUc! zpXzfpSoa<+-&8(FM2vqB31mRo zP>3{5Ln@Vo5CTL5W}UM0IzWut+PXFMR_k#=;`qpjXa3IJ+ru&(o*_-spv`BY2nitw z0D(XdM7+FkCHZ_lblvETjg5{US-ontTgKz@pDQzbBOYHChjn4I)bcvX~E*TNJEi{x^R5gQ->95gt(i;9R z4nS_D&C>61wD$Rym%pO0u+RViR;zXIAaAvnH#?k{y&jL7)*`DE6n7#XD=umjOePcA zsx0vR2&pP6EuC#`SDWljc9kMSfvv4A0sxW-9M1y)6pD07(C6yjI9i<*rRGv@b8{Wr zi3GqAqOk}VEpbIf#pn8-KWpgj>GJw~0q8V2NRlOyJUIoaDQQro zXF!>e31vnmWHJG{dOfkh!9ktfZeRHu=6?IVLYYjsnP)U!X+D1u>zgr12?-qIu(-5{ z7ccwIGPm2UEh#DS??*;N_&74GoSU1iRco|QYR}XeBasN+5522re!o9WtJTI1VogL~ jjP2GRMuyDlY_0D%<{TiF_>Qx&00000NkvXXu0mjfk8WAP literal 0 HcmV?d00001 diff --git a/modules/webstyle/img/yoursearches_search.png b/modules/webstyle/img/yoursearches_search.png new file mode 100644 index 0000000000000000000000000000000000000000..742d87d0eea3ba912475ce8cca9909ff3ccdf13f GIT binary patch literal 803 zcmV+;1Kj+HP)o%n$G|L0-#AmK~y-)osVlsQ*juFfB$oiZo0YcY|5tQWois;n1Q57T4GX$ z%m}>X1#Id>G?bM5qS2Qikj$VP5>iw=Y4oz@H`JM-MWM0umzSh$R4U{7OgrQ=&40fK-bt zSy%Nrx_`~IV_9*Y%tB;DDM>;I3$sx1sX!@+92_=&UY9iJPvumU>2m?d=Egetl-ZH}6@qrHLSnb9NPd3rKHt>%Y^xBdt*Q)MjgG(A!GxLIe*QKAMF>SG za6HeuJUm>BOFk`XPtlp8ymSDLjRBfgDC610LkAqJ%B*2UB!(3sN+v>>SV&YN0i=N} z_hD|)+H@o_=4D_1=*9q}mP@}%mS*j7v7w?+C2N!VFc8o*oE!&>o8!OCE%+r%Txr!6 z4Tm>-fgYcju#fchbnl}aA!_10?eDY;Wt@Y-!nDTE{60I0vKDHi)!{5F|6w#5O@OZ( z;ILB^rjDYiU%kbASG{apF2o-E55@6Fo$sn#Sb44IpVSAAWoZ#j3002ovPDHLkV1msIXo&y- literal 0 HcmV?d00001 diff --git a/modules/webstyle/img/yoursearches_search_last_performed.png b/modules/webstyle/img/yoursearches_search_last_performed.png new file mode 100644 index 0000000000000000000000000000000000000000..b52ed5bb71c6937ab451a571eb6d476aae1e34bd GIT binary patch literal 661 zcmV;G0&4wZ4IEgqMPCUtbb!g7PXnVs8FVAy7pZ5cQkpLuT zXJ-OT0lWwyIB+wi+G@4zcDud+NYm7jkr8^m9wP(RCGZM3p<1nm5Q3^ek9NE5;^LyJ zQ4|Hn$H!G&UtbqE%PJjItJR>|>2y>b8ym~pxKz#ZZfS$!L6;Wp+lDN9MnrDjR z*vZMsp&iub=H`r|DAa1TzbcVSIfSo3Ix{ol?(VLQM#E;aX{A!JSS$*pz`;KQ3x$Gl z9Q*L_;L6I1s!5WVB#EkPYir)#-kKyyCiJ&lf$8aKRZFGP(4ZF=7uM_bAzQgzHceA! zXJ-XI0>q6*!zhaKC1hAyT5@@L*{8>HeSK}G(-8;&SCvX7C!}fW#>R%*+uP4B>~Vp^ z!$X1J0MkHve0&`4@9&=lJ^kO^-Fbd~9tKLheZ0uiiHQkMPftT8dK5*r+ig|5-LCa| vJq+~y0leftcmsR_egO_%1NZ{GdrJNT%MwrGizb5800000NkvXXu0mjf6N4YE literal 0 HcmV?d00001 From a8f749d7225495be4de6c65e0d75618ee4aec9df Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Tue, 26 Jul 2011 14:06:22 +0200 Subject: [PATCH 18/59] WebAlert: youralerts interface improvements * Redesigns the "Your Alerts" display interface. * Adds new functionality to display alerts based on a specific search query. (closes #882) --- modules/webalert/lib/webalert.py | 94 ++-- modules/webalert/lib/webalert_templates.py | 408 +++++++++++++----- modules/webalert/lib/webalert_webinterface.py | 16 +- modules/websearch/lib/websearch_templates.py | 68 +-- modules/webstyle/css/invenio.css | 122 +++++- modules/webstyle/img/youralerts_alert.png | Bin 0 -> 991 bytes .../webstyle/img/youralerts_alert_dates.png | Bin 0 -> 661 bytes .../webstyle/img/youralerts_alert_delete.png | Bin 0 -> 1160 bytes .../webstyle/img/youralerts_alert_edit.png | Bin 0 -> 1213 bytes .../webstyle/img/youralerts_alert_search.png | Bin 0 -> 803 bytes 10 files changed, 531 insertions(+), 177 deletions(-) create mode 100644 modules/webstyle/img/youralerts_alert.png create mode 100644 modules/webstyle/img/youralerts_alert_dates.png create mode 100644 modules/webstyle/img/youralerts_alert_delete.png create mode 100644 modules/webstyle/img/youralerts_alert_edit.png create mode 100644 modules/webstyle/img/youralerts_alert_search.png diff --git a/modules/webalert/lib/webalert.py b/modules/webalert/lib/webalert.py index 8f8e3a38c9..67b0366139 100644 --- a/modules/webalert/lib/webalert.py +++ b/modules/webalert/lib/webalert.py @@ -173,59 +173,88 @@ def perform_add_alert(alert_name, frequency, notification, run_sql(query, params) out = _("The alert %s has been added to your profile.") out %= '' + cgi.escape(alert_name) + '' - out += perform_request_youralerts_display(uid, ln=ln) + out += perform_request_youralerts_display(uid, idq=None, ln=ln) return out def perform_request_youralerts_display(uid, + idq=None, ln=CFG_SITE_LANG): """ - Display a list of the user defined alerts. + Display a list of the user defined alerts. If a specific query id is defined + only the user alerts based on that query appear. + @param uid: The user id @type uid: int + + @param idq: The specified query id for which to display the user alerts + @type idq: int + @param ln: The interface language @type ln: string + @return: HTML formatted list of the user defined alerts. """ # set variables out = "" + if idq: + idq_clause = "AND uqb.id_query=%i" % (idq,) + else: + idq_clause = "" + # query the database - query = """ SELECT q.id, q.urlargs, - a.id_basket, b.name, - a.alert_name, a.frequency,a.notification, - DATE_FORMAT(a.date_creation,'%%Y-%%m-%%d %%H:%%i:%%s'), - DATE_FORMAT(a.date_lastrun,'%%Y-%%m-%%d %%H:%%i:%%s') - FROM user_query_basket a LEFT JOIN query q ON a.id_query=q.id - LEFT JOIN bskBASKET b ON a.id_basket=b.id - WHERE a.id_user=%s - ORDER BY a.alert_name ASC """ - res = run_sql(query, (uid,)) + query = """ SELECT q.id, + q.urlargs, + uqb.id_basket, + bsk.name, + uqb.alert_name, + uqb.frequency, + uqb.notification, + DATE_FORMAT(uqb.date_creation,'%s'), + DATE_FORMAT(uqb.date_lastrun,'%s') + FROM user_query_basket uqb + LEFT JOIN query q + ON uqb.id_query=q.id + LEFT JOIN bskBASKET bsk + ON uqb.id_basket=bsk.id + WHERE uqb.id_user=%%s + %s + ORDER BY uqb.alert_name ASC""" % ('%%Y-%%m-%%d %%H:%%i:%%s', + '%%Y-%%m-%%d %%H:%%i:%%s', + idq_clause,) + params = (uid,) + result = run_sql(query, params) + alerts = [] - for (qry_id, qry_args, - bsk_id, bsk_name, - alrt_name, alrt_frequency, alrt_notification, alrt_creation, alrt_last_run) in res: + for (query_id, + query_args, + bsk_id, + bsk_name, + alert_name, + alert_frequency, + alert_notification, + alert_creation, + alert_last_run) in result: try: - if not qry_id: + if not query_id: raise StandardError("""\ Warning: I have detected a bad alert for user id %d. It seems one of his/her alert queries was deleted from the 'query' table. Please check this and delete it if needed. Otherwise no problem, I'm continuing with the other alerts now. -Here are all the alerts defined by this user: %s""" % (uid, repr(res))) - alerts.append({ - 'queryid' : qry_id, - 'queryargs' : qry_args, - 'textargs' : get_textual_query_info_from_urlargs(qry_args, ln=ln), - 'userid' : uid, - 'basketid' : bsk_id, - 'basketname' : bsk_name, - 'alertname' : alrt_name, - 'frequency' : alrt_frequency, - 'notification' : alrt_notification, - 'created' : convert_datetext_to_dategui(alrt_creation), - 'lastrun' : convert_datetext_to_dategui(alrt_last_run) - }) +Here are all the alerts defined by this user: %s""" % (uid, repr(result))) + alerts.append({'queryid' : query_id, + 'queryargs' : query_args, + 'textargs' : get_textual_query_info_from_urlargs(query_args, ln=ln), + 'userid' : uid, + 'basketid' : bsk_id, + 'basketname' : bsk_name, + 'alertname' : alert_name, + 'frequency' : alert_frequency, + 'notification' : alert_notification, + 'created' : alert_creation, + 'lastrun' : alert_last_run}) except StandardError: register_exception(alert_admin=True) @@ -233,6 +262,7 @@ def perform_request_youralerts_display(uid, out = webalert_templates.tmpl_youralerts_display(ln=ln, alerts=alerts, guest=isGuestUser(uid), + idq=idq, guesttxt=warning_guest_user(type="alerts", ln=ln)) return out @@ -261,7 +291,7 @@ def perform_remove_alert(alert_name, id_query, id_basket, uid, ln=CFG_SITE_LANG) out += "The alert %s has been removed from your profile.

\n" % cgi.escape(alert_name) else: out += "Unable to remove alert %s.

\n" % cgi.escape(alert_name) - out += perform_request_youralerts_display(uid, ln=ln) + out += perform_request_youralerts_display(uid, idq=None, ln=ln) return out @@ -321,7 +351,7 @@ def perform_update_alert(alert_name, frequency, notification, id_basket, id_quer run_sql(query, params) out += _("The alert %s has been successfully updated.") % ("" + cgi.escape(alert_name) + "",) - out += "

\n" + perform_request_youralerts_display(uid, ln=ln) + out += "

\n" + perform_request_youralerts_display(uid, idq=None, ln=ln) return out def is_selected(var, fld): diff --git a/modules/webalert/lib/webalert_templates.py b/modules/webalert/lib/webalert_templates.py index 782b94d163..f8018fd675 100644 --- a/modules/webalert/lib/webalert_templates.py +++ b/modules/webalert/lib/webalert_templates.py @@ -19,19 +19,23 @@ import cgi import time +from urlparse import parse_qs +from datetime import date as datetime_date from invenio.config import \ CFG_WEBALERT_ALERT_ENGINE_EMAIL, \ CFG_SITE_NAME, \ CFG_SITE_SUPPORT_EMAIL, \ CFG_SITE_URL, \ + CFG_SITE_LANG, \ + CFG_SITE_SECURE_URL, \ CFG_WEBALERT_MAX_NUM_OF_RECORDS_IN_ALERT_EMAIL, \ CFG_SITE_RECORD from invenio.messages import gettext_set_language from invenio.htmlparser import get_as_text, wrap, wrap_records from invenio.urlutils import create_html_link - from invenio.search_engine import guess_primary_collection_of_a_record, get_coll_ancestors +from invenio.dateutils import convert_datetext_to_datestruct class Template: def tmpl_errorMsg(self, ln, error_msg, rest = ""): @@ -260,10 +264,13 @@ def tmpl_input_alert(self, ln, query, alert_name, action, frequency, notificatio def tmpl_youralerts_display(self, ln, alerts, + idq, guest, guesttxt): """ - Displays the list of alerts + Displays an HTML formatted list of the user alerts. + If the user has specified a query id, only the user alerts based on that + query will appear. @param ln: The language to display the interface in @type ln: string @@ -282,6 +289,9 @@ def tmpl_youralerts_display(self, 'lastrun' *string* - The last running date @type alerts: list of dictionaries + @param idq: The specified query id for which to display the user alerts + @type idq: int + @param guest: Whether the user is a guest or not @type guest: boolean @@ -293,108 +303,153 @@ def tmpl_youralerts_display(self, # load the right message language _ = gettext_set_language(ln) - out = '

' + _("Set a new alert from %(x_url1_open)syour searches%(x_url1_close)s, the %(x_url2_open)spopular alerts%(x_url2_close)s, or the input form.") + '

' - out %= {'x_url1_open': '', - 'x_url1_close': '', - 'x_url2_open': '', - 'x_url2_close': '', - } - if len(alerts): - out += """ - - - - - - - - - - """ % { - 'no' : _("No"), - 'name' : _("Name"), - 'search_freq' : _("Search checking frequency"), - 'notification' : _("Notification by email"), - 'result_basket' : _("Result in basket"), - 'date_run' : _("Date last run"), - 'date_created' : _("Creation date"), - 'query' : _("Query"), - 'action' : _("Action"), - } - i = 0 - for alert in alerts: - i += 1 - if alert['frequency'] == "day": - frequency = _("daily") - else: - if alert['frequency'] == "week": - frequency = _("weekly") - else: - frequency = _("monthly") + # In case the user has not yet defined any alerts display only the + # following message + if not alerts: + if idq: + msg = _('You have not defined any alerts yet based on that search query.') + msg += "
" + msg += _('You may want to %(new_alert)s or display all %(youralerts)s.') % \ + {'new_alert': '%s' % (CFG_SITE_SECURE_URL, ln, idq, _('define one now')), + 'youralerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('your alerts'))} + else: + msg = _('You have not defined any alerts yet.') + msg += '
' + msg += _('You may define new alert based on %(yoursearches)s, the %(popular_alerts)s or just by %(search_interface)s.') % \ + {'yoursearches': '%s' % (CFG_SITE_SECURE_URL, ln, _('your searches')), + 'popular_alerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('popular alerts')), + 'search_interface': '%s' %(CFG_SITE_URL, ln, _('searching for something new'))} + out = '

' + msg + '

' + return out + + # Diplay a message about the number of alerts. + if idq: + msg = _('You have defined %(number_of_alerts)s alerts based on that search query.') % \ + {'number_of_alerts': '' + str(len(alerts)) + ''} + msg += '
' + msg += _('You may want to %(new_alert)s or display all %(youralerts)s.') % \ + {'new_alert': '%s' % (CFG_SITE_SECURE_URL, ln, idq, _('define a new one')), + 'youralerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('your alerts'))} + else: + msg = _('You have defined a total of %(number_of_alerts)s alerts.') % \ + {'number_of_alerts': '' + str(len(alerts)) + ''} + msg += '
' + msg += _('You may define new alert based on %(yoursearches)s, the %(popular_alerts)s or just by %(search_interface)s.') % \ + {'yoursearches': '%s' % (CFG_SITE_SECURE_URL, ln, _('your searches')), + 'popular_alerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('popular alerts')), + 'search_interface': '%s' %(CFG_SITE_URL, ln, _('searching for something new'))} + out = '

' + msg + '

' + + counter = 0 + youralerts_display_html = "" + for alert in alerts: + counter += 1 + alert_name = alert['alertname'] + alert_query_id = alert['queryid'] + alert_query_args = alert['queryargs'] + # We don't need the text args, we'll use a local function to do a + # better job. + #alert_text_args = alert['textargs'] + # We don't need the user id. The alerts page is a logged in user + # only page anyway. + #alert_user_id = alert['userid'] + alert_basket_id = alert['basketid'] + alert_basket_name = alert['basketname'] + alert_frequency = alert['frequency'] + alert_notification = alert['notification'] + alert_creation_date = alert['created'] + alert_last_run_date = alert['lastrun'] + + alert_details_frequency = _('Runs') + ' ' + \ + (alert_frequency == 'day' and '' + _('daily') + '' or \ + alert_frequency == 'week' and '' + _('weekly') + '' or \ + alert_frequency == 'month' and '' + _('monthly') + '') + alert_details_notification = alert_notification == 'y' and _('You are notified by e-mail') or \ + alert_notification == 'n' and '' + alert_details_basket = alert_basket_name and _('The results are automatically added to your basket:') + \ + ' ' + '' + cgi.escape(alert_basket_name) + '' or '' + alert_details_frequency_notification_basket = alert_details_frequency + \ + (alert_details_notification and \ + ' / ' + \ + alert_details_notification) + \ + (alert_details_basket and \ + ' / ' + \ + alert_details_basket) + + alert_details_search_query = get_html_user_friendly_alert_query_args(alert_query_args, ln) + + alert_details_creation_date = get_html_user_friendly_date_from_datetext(alert_creation_date, True, False, ln) + alert_details_last_run_date = get_html_user_friendly_date_from_datetext(alert_last_run_date, True, False, ln) + alert_details_creation_last_run_dates = _('Created:') + ' ' + \ + alert_details_creation_date + \ + ' / ' + \ + _('Last run:') + ' ' + \ + alert_details_last_run_date + + alert_details_options_edit = create_html_link('%s/youralerts/modify' % \ + (CFG_SITE_SECURE_URL,), + {'ln' : ln, + 'idq' : alert_query_id, + 'name' : alert_name, + 'freq' : alert_frequency, + 'notif' : alert_notification, + 'idb' : alert_basket_id, + 'old_idb': alert_basket_id}, + _('Edit')) + alert_details_options_delete = create_html_link('%s/youralerts/remove' % \ + (CFG_SITE_SECURE_URL,), + {'ln' : ln, + 'idq' : alert_query_id, + 'name' : alert_name, + 'idb' : alert_basket_id}, + _('Delete')) + alert_details_options = '' % (CFG_SITE_URL,) + \ + alert_details_options_edit + \ + '   ' + \ + '' % (CFG_SITE_URL,) + \ + alert_details_options_delete + + youralerts_display_html += """ + + + + """ % {'counter': counter, + 'alert_name': cgi.escape(alert_name), + 'alert_details_frequency_notification_basket': alert_details_frequency_notification_basket, + 'alert_details_search_query': alert_details_search_query, + 'alert_details_options': alert_details_options, + 'alert_details_creation_last_run_dates': alert_details_creation_last_run_dates} + + out += """ +
%(no)s%(name)s%(search_freq)s%(notification)s%(result_basket)s%(date_run)s%(date_created)s%(query)s%(action)s
+ %(counter)i. + +
+
%(alert_name)s
+
%(alert_details_frequency_notification_basket)s
+
%(alert_details_search_query)s
+
+
+
%(alert_details_options)s
+
+
+
%(alert_details_creation_last_run_dates)s
+
+ + + + + + + + + + + + %(youralerts_display_html)s + +
""" % {'youralerts_display_html': youralerts_display_html} - if alert['notification'] == "y": - notification = _("yes") - else: - notification = _("no") - - # we clean up the HH:MM part of lastrun, since it is always 00:00 - lastrun = alert['lastrun'].split(',')[0] - created = alert['created'].split(',')[0] - - out += """ - #%(index)d - %(alertname)s - %(frequency)s - %(notification)s - %(basketname)s - %(lastrun)s - %(created)s - %(textargs)s - - %(remove_link)s
- %(modify_link)s
- %(search)s - - """ % { - 'index' : i, - 'alertname' : cgi.escape(alert['alertname']), - 'frequency' : frequency, - 'notification' : notification, - 'basketname' : alert['basketname'] and cgi.escape(alert['basketname']) \ - or "- " + _("no basket") + " -", - 'lastrun' : lastrun, - 'created' : created, - 'textargs' : alert['textargs'], - 'queryid' : alert['queryid'], - 'basketid' : alert['basketid'], - 'freq' : alert['frequency'], - 'notif' : alert['notification'], - 'ln' : ln, - 'modify_link': create_html_link("./modify", - {'ln': ln, - 'idq': alert['queryid'], - 'name': alert['alertname'], - 'freq': frequency, - 'notif':notification, - 'idb':alert['basketid'], - 'old_idb':alert['basketid']}, - _("Modify")), - 'remove_link': create_html_link("./remove", - {'ln': ln, - 'idq': alert['queryid'], - 'name': alert['alertname'], - 'idb':alert['basketid']}, - _("Remove")), - 'siteurl' : CFG_SITE_URL, - 'search' : _("Execute search"), - 'queryargs' : cgi.escape(alert['queryargs']) - } - - out += '' - - out += '

' + (_("You have defined %s alerts.") % ('' + str(len(alerts)) + '' )) + '

' - if guest: - out += guesttxt return out def tmpl_alert_email_title(self, name): @@ -627,3 +682,150 @@ def tmpl_youralerts_popular(self, out += "
\n" return out + +def get_html_user_friendly_alert_query_args(args, + ln=CFG_SITE_LANG): + """ + Internal function. + Returns an HTML formatted user friendly description of a search query's + arguments. + + @param args: The search query arguments as they apear in the search URL + @type args: string + + @param ln: The language to display the interface in + @type ln: string + + @return: HTML formatted user friendly description of a search query's + arguments + """ + + # Load the right language + _ = gettext_set_language(ln) + + # Arguments dictionary + dict = parse_qs(args) + + if not dict.has_key('p') and not dict.has_key('p1') and not dict.has_key('p2') and not dict.has_key('p3'): + search_patterns_html = _('Searching for everything') + else: + search_patterns_html = _('Searching for') + ' ' + if dict.has_key('p'): + search_patterns_html += '' + cgi.escape(dict['p'][0]) + '' + if dict.has_key('f'): + search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(dict['f'][0]) + '' + if dict.has_key('p1'): + if dict.has_key('p'): + search_patterns_html += ' ' + _('and') + ' ' + search_patterns_html += '' + cgi.escape(dict['p1'][0]) + '' + if dict.has_key('f1'): + search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(dict['f1'][0]) + '' + if dict.has_key('p2'): + if dict.has_key('p') or dict.has_key('p1'): + if dict.has_key('op1'): + search_patterns_html += ' %s ' % (dict['op1'][0] == 'a' and _('and') or \ + dict['op1'][0] == 'o' and _('or') or \ + dict['op1'][0] == 'n' and _('and not') or + ', ',) + search_patterns_html += '' + cgi.escape(dict['p2'][0]) + '' + if dict.has_key('f2'): + search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(dict['f2'][0]) + '' + if dict.has_key('p3'): + if dict.has_key('p') or dict.has_key('p1') or dict.has_key('p2'): + if dict.has_key('op2'): + search_patterns_html += ' %s ' % (dict['op2'][0] == 'a' and _('and') or \ + dict['op2'][0] == 'o' and _('or') or \ + dict['op2'][0] == 'n' and _('and not') or + ', ',) + search_patterns_html += '' + cgi.escape(dict['p3'][0]) + '' + if dict.has_key('f3'): + search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(dict['f3'][0]) + '' + + if not dict.has_key('c') and not dict.has_key('cc'): + collections_html = _('in all the collections') + else: + collections_html = _('in the following collection(s)') + ': ' + if dict.has_key('c'): + collections_html += ', '.join('' + cgi.escape(collection) + '' for collection in dict['c']) + elif dict.has_key('cc'): + collections_html += '' + cgi.escape(dict['cc'][0]) + '' + + search_query_args_html = search_patterns_html + '
' + collections_html + + return search_query_args_html + + +def get_html_user_friendly_date_from_datetext(given_date, + show_full_date=True, + show_full_time=True, + ln=CFG_SITE_LANG): + """ + Internal function. + Returns an HTML formatted user friendly description of a search query's + last run date. + + @param given_date: The search query last run date in the following format: + '2005-11-16 15:11:57' + @type given_date: string + + @param show_full_date: show the full date as well + @type show_full_date: boolean + + @param show_full_time: show the full time as well + @type show_full_time: boolean + + @param ln: The language to display the interface in + @type ln: string + + @return: HTML formatted user friendly description of a search query's + last run date + """ + + # Load the right language + _ = gettext_set_language(ln) + + # Calculate how many days old the search query is base on the given date + # and today + # given_date_datestruct[0] --> year + # given_date_datestruct[1] --> month + # given_date_datestruct[2] --> day in month + given_date_datestruct = convert_datetext_to_datestruct(given_date) + today = datetime_date.today() + if given_date_datestruct[0] != 0 and \ + given_date_datestruct[1] != 0 and \ + given_date_datestruct[2] != 0: + days_old = (today - datetime_date(given_date_datestruct.tm_year, + given_date_datestruct.tm_mon, + given_date_datestruct.tm_mday)).days + if days_old == 0: + out = _('Today') + elif days_old < 7: + out = str(days_old) + ' ' + _('day(s) ago') + elif days_old == 7: + out = _('A week ago') + elif days_old < 14: + out = _('More than a week ago') + elif days_old == 14: + out = _('Two weeks ago') + elif days_old < 30: + out = _('More than two weeks ago') + elif days_old == 30: + out = _('A month ago') + elif days_old < 180: + out = _('More than a month ago') + elif days_old < 365: + out = _('More than six months ago') + else: + out = _('More than a year ago') + if show_full_date: + out += '' + \ + ' ' + _('on') + ' ' + \ + given_date.split()[0] + '' + if show_full_time: + out += '' + \ + ' ' + _('at') + ' ' + \ + given_date.split()[1] + '' + else: + out = _('Unknown') + + return out diff --git a/modules/webalert/lib/webalert_webinterface.py b/modules/webalert/lib/webalert_webinterface.py index a7afc94dbf..a189e9c7fd 100644 --- a/modules/webalert/lib/webalert_webinterface.py +++ b/modules/webalert/lib/webalert_webinterface.py @@ -239,7 +239,8 @@ def modify(self, req, form): def display(self, req, form): - argd = wash_urlargd(form, {}) + argd = wash_urlargd(form, {'idq': (int, None), + }) uid = getUid(req) @@ -274,12 +275,13 @@ def display(self, req, form): register_exception(suffix="Do the webstat tables exists? Try with 'webstatadmin --load-config'") return page(title=_("Your Alerts"), - body=perform_request_youralerts_display(uid, ln = argd['ln']), - navtrail= """%(account)s""" % { - 'sitesecureurl' : CFG_SITE_SECURE_URL, - 'ln': argd['ln'], - 'account' : _("Your Account"), - }, + body=perform_request_youralerts_display(uid, + idq=argd['idq'], + ln=argd['ln']), + navtrail= """%(account)s""" % \ + {'sitesecureurl' : CFG_SITE_SECURE_URL, + 'ln': argd['ln'], + 'account' : _("Your Account")}, description=_("%s Personalize, Display alerts") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME), keywords=_("%s, personalize") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME), uid=uid, diff --git a/modules/websearch/lib/websearch_templates.py b/modules/websearch/lib/websearch_templates.py index 68973a77f2..b8f762472b 100644 --- a/modules/websearch/lib/websearch_templates.py +++ b/modules/websearch/lib/websearch_templates.py @@ -29,6 +29,7 @@ from urllib import quote, urlencode from urlparse import parse_qs from xml.sax.saxutils import escape as xml_escape +from datetime import date as datetime_date from invenio.config import \ CFG_WEBSEARCH_LIGHTSEARCH_PATTERN_BOX_WIDTH, \ @@ -82,9 +83,7 @@ from invenio.webinterface_handler import wash_urlargd from invenio.bibrank_citation_searcher import get_cited_by_count from invenio.webuser import session_param_get - from invenio.intbitset import intbitset - from invenio.websearch_external_collections import external_collection_get_state, get_external_collection_engine from invenio.websearch_external_collections_utils import get_collection_id from invenio.websearch_external_collections_config import CFG_EXTERNAL_COLLECTION_MAXRESULTS @@ -95,9 +94,6 @@ from invenio import hepdatadisplayutils from invenio.dateutils import convert_datetext_to_datestruct -from datetime import date as datetime_date - -#from invenio.websearch_yoursearches import count_user_alerts_for_given_query _RE_PUNCTUATION = re.compile(CFG_BIBINDEX_CHARS_PUNCTUATION) _RE_SPACES = re.compile(r"\s+") @@ -5145,26 +5141,27 @@ def tmpl_yoursearches_display(self, # following message if not search_queries: if p: - out = _("You have not searched for anything yet including the terms %(p)s. You may perform that %(x_url_open)ssearch%(x_url_close)s for the first time now.") % \ + msg = _("You have not searched for anything yet including the terms %(p)s. You may perform that %(x_url_open)ssearch%(x_url_close)s for the first time now.") % \ {'p': '' + cgi.escape(p) + '', 'x_url_open': '', 'x_url_close': ''} else: - out = _("You have not searched for anything yet. You may want to start by the %(x_url_open)ssearch interface%(x_url_close)s first.") % \ + msg = _("You have not searched for anything yet. You may want to start by the %(x_url_open)ssearch interface%(x_url_close)s first.") % \ {'x_url_open': '', 'x_url_close': ''} + out = '

' + msg + '

' return out # Diplay a message about the number of searches. if p: msg = _("You have performed %(searches_distinct)s unique searches in a total of %(searches_total)s searches including the term %(p)s.") % \ - {'searches_distinct': nb_queries_distinct, - 'searches_total': nb_queries_total, + {'searches_distinct': '' + str(nb_queries_distinct) + '', + 'searches_total': '' + str(nb_queries_total) + '', 'p': '' + cgi.escape(p) + ''} else: msg = _("You have performed %(searches_distinct)s unique searches in a total of %(searches_total)s searches.") % \ - {'searches_distinct': nb_queries_distinct, - 'searches_total': nb_queries_total} + {'searches_distinct': '' + str(nb_queries_distinct) + '', + 'searches_total': '' + str(nb_queries_total) + ''} out = '

' + msg + '

' # Search form @@ -5191,7 +5188,7 @@ def tmpl_yoursearches_display(self, search_query_details = get_html_user_friendly_search_query_args(search_query_args, ln) - search_query_last_performed = get_html_user_friendly_search_query_lastrun(search_query_lastrun, ln) + search_query_last_performed = get_html_user_friendly_date_from_datetext(search_query_lastrun, ln) search_query_options_search = """%s""" % \ (CFG_SITE_SECURE_URL, cgi.escape(search_query_args), CFG_SITE_URL, _('Search again')) @@ -5337,16 +5334,24 @@ def get_html_user_friendly_search_query_args(args, return search_query_args_html -def get_html_user_friendly_search_query_lastrun(lastrun, - ln=CFG_SITE_LANG): +def get_html_user_friendly_date_from_datetext(given_date, + show_full_date=True, + show_full_time=True, + ln=CFG_SITE_LANG): """ Internal function. Returns an HTML formatted user friendly description of a search query's last run date. - @param lastrun: The search query last run date in the following format: + @param given_date: The search query last run date in the following format: '2005-11-16 15:11:57' - @type lastrun: string + @type given_date: string + + @param show_full_date: show the full date as well + @type show_full_date: boolean + + @param show_full_time: show the full time as well + @type show_full_time: boolean @param ln: The language to display the interface in @type ln: string @@ -5358,16 +5363,19 @@ def get_html_user_friendly_search_query_lastrun(lastrun, # Load the right language _ = gettext_set_language(ln) - # Calculate how many days old the search query is base on the lastrun date + # Calculate how many days old the search query is base on the given date # and today - lastrun_datestruct = convert_datetext_to_datestruct(lastrun) + # given_date_datestruct[0] --> year + # given_date_datestruct[1] --> month + # given_date_datestruct[2] --> day in month + given_date_datestruct = convert_datetext_to_datestruct(given_date) today = datetime_date.today() - if lastrun_datestruct.tm_year != 0 and \ - lastrun_datestruct.tm_mon != 0 and \ - lastrun_datestruct.tm_mday != 0: - days_old = (today - datetime_date(lastrun_datestruct.tm_year, - lastrun_datestruct.tm_mon, - lastrun_datestruct.tm_mday)).days + if given_date_datestruct[0] != 0 and \ + given_date_datestruct[1] != 0 and \ + given_date_datestruct[2] != 0: + days_old = (today - datetime_date(given_date_datestruct.tm_year, + given_date_datestruct.tm_mon, + given_date_datestruct.tm_mday)).days if days_old == 0: out = _('Today') elif days_old < 7: @@ -5388,10 +5396,14 @@ def get_html_user_friendly_search_query_lastrun(lastrun, out = _('More than six months ago') else: out = _('More than a year ago') - out += '' + \ - ' ' + _('on') + ' ' + lastrun.split()[0] + \ - ' ' + _('at') + ' ' + lastrun.split()[1] + \ - '' + if show_full_date: + out += '' + \ + ' ' + _('on') + ' ' + \ + given_date.split()[0] + '' + if show_full_time: + out += '' + \ + ' ' + _('at') + ' ' + \ + given_date.split()[1] + '' else: out = _('Unknown') diff --git a/modules/webstyle/css/invenio.css b/modules/webstyle/css/invenio.css index b8753925a5..f03c01ca69 100644 --- a/modules/webstyle/css/invenio.css +++ b/modules/webstyle/css/invenio.css @@ -2580,6 +2580,120 @@ a:hover.bibMergeImgClickable img { border: 1px solid black; padding: 3px; } + +.youralerts_display_table { + border: 1px solid #bbbbbb; +} +.youralerts_display_table_header td { + background-color: #7a9bdd; + padding: 2px; + border-bottom: 1px solid #3366cc; + border-right: 1px solid #3366cc; +} +.youralerts_display_table_footer td { + background-color: #7a9bdd; + color: white; + padding: 2px; + font-size: 80%; + text-align: center; + border-bottom: 1px solid #3366cc; + border-right: 1px solid #3366cc; + white-space: nowrap; +} +.youralerts_display_table_footer img { + border: none; + margin: 0px 3px; + vertical-align: bottom; +} +.youralerts_display_footer a, .youralerts_display_footer a:link, .youralerts_display_footer a:visited, .youralerts_display_footer a:active { + text-decoration: none; + color: #333; +} +.youralerts_display_footer a:hover { + text-decoration: underline; + color: #000; +} +.youralerts_display_footer img { + border: none; + vertical-align: bottom; +} +.youralerts_display_table_content { + background-color: #f2f2fa; + padding: 5px; + text-align: left; + vertical-align: top; +} +.youralerts_display_table_content_mouseover { + background-color: #ebebfa; + padding: 5px; + text-align: left; + vertical-align: top; +} +.youralerts_display_table_content_container_left { + float: left; + width: 75%; +} +.youralerts_display_table_content_container_right { + float: right; + width: 25%; +} +.youralerts_display_table_content_clear { + clear: both; +} +.youralerts_display_table_counter { + background-color: #f2f2fa; + padding: 5px 2px; + text-align: right; + vertical-align: top; + font-size: 80%; + color: gray; +} +.youralerts_display_table_content_name { + margin-bottom: 5px; + font-weight: bold; + position: relative; + left: 0px; + top: 0px; +} +.youralerts_display_table_content_details { + margin-bottom: 5px; + font-size: 80%; + position: relative; + left: 0px; +} +.youralerts_display_table_content_search_query { + margin-bottom: 5px; + font-size: 80%; + position: relative; + left: 0px; +} +.youralerts_display_table_content_dates { + position: relative; + font-size: 70%; + text-align: right; + white-space: nowrap; + color: gray; +} +.youralerts_display_table_content_options { + position: relative; + top: 0px; + right: 0px; + margin-bottom: 5px; + font-size: 80%; + text-align: right; +} +.youralerts_display_table_content_options img{ + vertical-align: top; + margin-right: 3px; +} +.youralerts_display_table_content_options a, .youralerts_display_table_content_options a:link, .youralerts_display_table_content_options a:visited, .youralerts_display_table_content_options a:active { + text-decoration: underline; + color: #333; +} +.youralerts_display_table_content_options a:hover { + text-decoration: underline; + color: #000; +} /* end of WebAlert module */ /* WebSearch module - YourSearches */ @@ -2587,29 +2701,23 @@ a:hover.bibMergeImgClickable img { border: 1px solid #bbbbbb; } .websearch_yoursearches_table_header td { -/* background-color: #63a5cd;*/ background-color: #7a9bdd; color: white; padding: 2px; font-weight: bold; font-size: 75%; text-transform: uppercase; -/* border-bottom: 1px solid #327aa5; - border-right: 1px solid #327aa5;*/ border-bottom: 1px solid #3366cc; border-right: 1px solid #3366cc; white-space: nowrap; vertical-align: top; } .websearch_yoursearches_table_footer td { -/* background-color: #63a5cd;*/ background-color: #7a9bdd; color: white; padding: 5px; font-size: 80%; text-align: center; -/* border-bottom: 1px solid #327aa5; - border-right: 1px solid #327aa5;*/ border-bottom: 1px solid #3366cc; border-right: 1px solid #3366cc; white-space: nowrap; @@ -2656,7 +2764,7 @@ a:hover.bibMergeImgClickable img { font-size: 80%; } .websearch_yoursearches_table_content_options img{ - vertical-align: middle; + vertical-align: top; margin-right: 3px; } .websearch_yoursearches_table_content_options a, .websearch_yoursearches_table_content_options a:link, .websearch_yoursearches_table_content_options a:visited, .websearch_yoursearches_table_content_options a:active { diff --git a/modules/webstyle/img/youralerts_alert.png b/modules/webstyle/img/youralerts_alert.png new file mode 100644 index 0000000000000000000000000000000000000000..c036ff6de4ff10cfb3b89b0c52740898b78639ee GIT binary patch literal 991 zcmV<510ei~P)w5Rx83Uv#ml(IHCMomsEWAF^>~ z?zaaWadt7U(R1e^AIE<%`o;KfUq0*R`n8>YAge7Iv+Z=3}v#78e&) z3JVJhipS#`85v0xPb=y0@KCv2PN7gph0@l>D^m1D{r&?wO|UGhB>NL}m!(ZkABV|~ z!TIR3M)UYqgk&=LNK1E@o|iU`75cvaSXLvS&nutL<8(S7 zop{@Te@@;X2crq2Y<_;;E+x_I>>bCQJ~*)pCBa+8m*@KMc`v&8ER>l*6Xc@M{}Wkb zdU`tB(9lp84u|RJ=%BH&5!<#o&zFbca5&iB-e!4unUSLLBqT|@ySutjSE10?WlH|y)>uEPsjf)B`r5Vn zk_8IHnM_8pSgh#V!^OvMPwBG{Ov(OR+VY;RgS~;SzQE}S=2;0+sT2nX2e@1=d_Erl zhEeQm;rz|FWAwZsxLXc+3yeY>hCc=%1Yk-s5{U%aY!;8lLv?j^N#e7(cmDx(?w|2q z5T#b3;S#+67B2ypjDdlH3ez;}|I5aBJf8eVxB74OUEhB9y(^$YFtR`%coL`wemlM@ zC`VuXh+**m@Z9A_Zulzbto3m5VlBpFE;!7Z4IEgqMPCUtbb!g7PXnVs8FVAy7pZ5cQkpLuT zXJ-OT0lWwyIB+wi+G@4zcDud+NYm7jkr8^m9wP(RCGZM3p<1nm5Q3^ek9NE5;^LyJ zQ4|Hn$H!G&UtbqE%PJjItJR>|>2y>b8ym~pxKz#ZZfS$!L6;Wp+lDN9MnrDjR z*vZMsp&iub=H`r|DAa1TzbcVSIfSo3Ix{ol?(VLQM#E;aX{A!JSS$*pz`;KQ3x$Gl z9Q*L_;L6I1s!5WVB#EkPYir)#-kKyyCiJ&lf$8aKRZFGP(4ZF=7uM_bAzQgzHceA! zXJ-XI0>q6*!zhaKC1hAyT5@@L*{8>HeSK}G(-8;&SCvX7C!}fW#>R%*+uP4B>~Vp^ z!$X1J0MkHve0&`4@9&=lJ^kO^-Fbd~9tKLheZ0uiiHQkMPftT8dK5*r+ig|5-LCa| vJq+~y0leftcmsR_egO_%1NZ{GdrJNT%MwrGizb5800000NkvXXu0mjf6N4YE literal 0 HcmV?d00001 diff --git a/modules/webstyle/img/youralerts_alert_delete.png b/modules/webstyle/img/youralerts_alert_delete.png new file mode 100644 index 0000000000000000000000000000000000000000..6d28c094b7e9144e2de856ffb0731630b1516b29 GIT binary patch literal 1160 zcmV;31b6$1P)ib)Zs-nt&78&?*xZ>_BmAXA`T0I<1;4iGO2a z^8YyRE;{X86}I2qIlS+4&U>Ecu+GlTiTwQhcK`rhuU8`?+PratSbu-N1_1GR{2_LB zc8G{*nM`KO=kpZ+09Vt3cF{kDja!Y zWOkMcY&Hh3*Q52>_4)z~yqGp`iikbQ-cOV`yjyZ>)x3 zom;@$xw(+eRp5iUd0YLJD;!?0SBu4BN2+J`QnY&cGOaZ=(dwm31VF%Xyakv8Wo2~F z?NR~Yp4+7d%F2j!bad!Wr;~{2$VNdB5Dte0K@jAYmKF{oqJe<{ip65YIX`rzSS&_^ zgM&mw)ZX5n2LRA@ojB)|PN%7-r{|!!r@NbipM83;UTC;54ggHobrMApM1*KG3Pn*M z$};4Z7RVkCQYI0k>oBVZj7a!-Ln4uY&1M5*3`B&D(H}uQcOl#DAR^H2E{%^2)-uB| z$S@4Zvi!op59;1SUQQlBgvi#qa&G3ngZX?u@D~n$qI$&Q| zRaGd80+-8mu(!{Xfj+E#@fBXFsDPB43(;(bBuR*Et|QM~2whcSq*5@FNrZk~*gN!l z5{U%1x3^(3nc#FfpZU!W`|4@>2-QnV;2(d8jimszXcXya6jqZNnRpypJdUT27E_bc zKV7YPyljz(h$)JKoSYok?e^zEVn+vtF^1&RHKe|~19z4MS|kF9;#=V1(qz|5JoNt2=^DCe9T zbFQSJHC^RGnfPB@vp@Qy%|Bdi*jk1g%jJ;li`}`{a8vXgGo7f1fWHQGf_kE!-W>-V z-mmXH{RLAa*Xz&E_PjsO`~5tR+P{Cl+itg40$>dCS zX0y@S`rD*33?ornTdRA$UQE-(W=o^<sM&%(4j*K zo6W{F@pzogn>Sk#5$Wpc5)qN^^W7pMa<>1bkV>Uwn!%v~IdS@&0O&54%bb=OdR(Zz zl0KTccI}#A7)HYH_e)1dhdhhVcSqXW|CEOhA4+p`^ThfOKDX)sd3kwMS635_MzLD0 zw70iYT3X6vWP;MtH*h!{*z=3C)@`uw!Z3_PFc_5CXgl5~pYAy+SFZMp&*u}KL)wSm zbjZ4|jtY*3Q>P|XQ&UsY;c$q^>;jsm;rIJBP1CIP_4O%4M2;LeBEeu#o{Y>i zO&J*(7N^rGA|g#c@5|Y+ahru?GAXH4N}|!Iw6wI$6tuRs%7u&F(to8-L`04rKRed$ zz9=f0Op3)~K|}}y0))e1B2y7GO(P>cgHR|)L18iOzb<$N2Zt+W3MM8d$jZt>DFwj2 z`=cZiNutpt1KwNQxq02bqhV)AnWibGX=1fn|7CdVcc&QjjdEbuYR+Fcw{K)*c&BnY zo#wLJ7!I??Bm9RmD_$1d+C1#flml z8bScNrfF1ER8Un_g@`abJWOY2C(D;Fr@XxUDY5BKP1EELXB$ROJmuquig@bI9<#HON2blobE zsc`Yim6gZF!{ccdT~qu|EWrGa!ybQbuo%n$G|L0-#AmK~y-)osVlsQ*juFfB$oiZo0YcY|5tQWois;n1Q57T4GX$ z%m}>X1#Id>G?bM5qS2Qikj$VP5>iw=Y4oz@H`JM-MWM0umzSh$R4U{7OgrQ=&40fK-bt zSy%Nrx_`~IV_9*Y%tB;DDM>;I3$sx1sX!@+92_=&UY9iJPvumU>2m?d=Egetl-ZH}6@qrHLSnb9NPd3rKHt>%Y^xBdt*Q)MjgG(A!GxLIe*QKAMF>SG za6HeuJUm>BOFk`XPtlp8ymSDLjRBfgDC610LkAqJ%B*2UB!(3sN+v>>SV&YN0i=N} z_hD|)+H@o_=4D_1=*9q}mP@}%mS*j7v7w?+C2N!VFc8o*oE!&>o8!OCE%+r%Txr!6 z4Tm>-fgYcju#fchbnl}aA!_10?eDY;Wt@Y-!nDTE{60I0vKDHi)!{5F|6w#5O@OZ( z;ILB^rjDYiU%kbASG{apF2o-E55@6Fo$sn#Sb44IpVSAAWoZ#j3002ovPDHLkV1msIXo&y- literal 0 HcmV?d00001 From f9c81d3ef451eedc902b9517f3a4fbc7f8697085 Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Tue, 23 Aug 2011 16:46:35 +0200 Subject: [PATCH 19/59] WebSearch: fix yoursearches string format bug * Fixes a small bug that caused special characters that had been replaced by the urllib.quote() function to crash during the string formatting. (closes #883) --- modules/websearch/lib/websearch_yoursearches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/websearch/lib/websearch_yoursearches.py b/modules/websearch/lib/websearch_yoursearches.py index 6df63ad3d3..b56a9f22f3 100644 --- a/modules/websearch/lib/websearch_yoursearches.py +++ b/modules/websearch/lib/websearch_yoursearches.py @@ -64,7 +64,7 @@ def perform_request_yoursearches_display(uid, if p: p_stripped = p.strip() p_stripped_args = p.split() - sql_p_stripped_args = ['\'%%' + quote(p_stripped_arg) + '%%\'' for p_stripped_arg in p_stripped_args] + sql_p_stripped_args = ['\'%%' + quote(p_stripped_arg).replace('%','%%') + '%%\'' for p_stripped_arg in p_stripped_args] for sql_p_stripped_arg in sql_p_stripped_args: search_clause += """ AND q.urlargs LIKE %s""" % (sql_p_stripped_arg,) From 1d5f4f6423f3c15fcc90325a1e2d46f3bc29fe1f Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Tue, 30 Aug 2011 14:34:37 +0200 Subject: [PATCH 20/59] WebSearch & WebAlert: CSS fallback to default * Changes some CSS attributes to match yoursearches and youralerts style to the default Invenio style. * Changes the order of the Personalize menu to be in alphabetical order again after the addition of Your Searches. * Deletes some previously added unneeded images. (addresses #884) --- modules/webalert/lib/webalert_templates.py | 2 +- modules/websearch/lib/websearch_templates.py | 2 +- .../websession/lib/websession_templates.py | 12 ++--- modules/webstyle/css/invenio.css | 46 +++++++++--------- .../webstyle/img/youralerts_alert_dates.png | Bin 661 -> 0 bytes .../webstyle/img/youralerts_alert_search.png | Bin 803 -> 0 bytes .../yoursearches_search_last_performed.png | Bin 661 -> 0 bytes 7 files changed, 31 insertions(+), 31 deletions(-) delete mode 100644 modules/webstyle/img/youralerts_alert_dates.png delete mode 100644 modules/webstyle/img/youralerts_alert_search.png delete mode 100644 modules/webstyle/img/yoursearches_search_last_performed.png diff --git a/modules/webalert/lib/webalert_templates.py b/modules/webalert/lib/webalert_templates.py index f8018fd675..957be0a36d 100644 --- a/modules/webalert/lib/webalert_templates.py +++ b/modules/webalert/lib/webalert_templates.py @@ -434,7 +434,7 @@ def tmpl_youralerts_display(self, 'alert_details_creation_last_run_dates': alert_details_creation_last_run_dates} out += """ - +
diff --git a/modules/websearch/lib/websearch_templates.py b/modules/websearch/lib/websearch_templates.py index b8f762472b..e5dca4571b 100644 --- a/modules/websearch/lib/websearch_templates.py +++ b/modules/websearch/lib/websearch_templates.py @@ -5242,7 +5242,7 @@ def tmpl_yoursearches_display(self, (CFG_SITE_SECURE_URL, paging_navigation[3], step, cgi.escape(p), ln, '/img/yoursearches_last_page.png') out += """ -
+
diff --git a/modules/websession/lib/websession_templates.py b/modules/websession/lib/websession_templates.py index e0179d6ff9..91d89f4711 100644 --- a/modules/websession/lib/websession_templates.py +++ b/modules/websession/lib/websession_templates.py @@ -1476,18 +1476,18 @@ def tmpl_create_useractivities_menu(self, ln, selected, url_referer, guest, user 'ln' : ln, 'messages' : _('Your messages') } - if submitter: - out += '
  • %(submissions)s
    • ' % { - 'CFG_SITE_SECURE_URL' : CFG_SITE_SECURE_URL, - 'ln' : ln, - 'submissions' : _('Your submissions') - } if usealerts or guest: out += '
  • %(searches)s
    • ' % { 'CFG_SITE_SECURE_URL' : CFG_SITE_SECURE_URL, 'ln' : ln, 'searches' : _('Your searches') } + if submitter: + out += '
  • %(submissions)s
    • ' % { + 'CFG_SITE_SECURE_URL' : CFG_SITE_SECURE_URL, + 'ln' : ln, + 'submissions' : _('Your submissions') + } out += '' return out diff --git a/modules/webstyle/css/invenio.css b/modules/webstyle/css/invenio.css index f03c01ca69..2fdbeeb1f5 100644 --- a/modules/webstyle/css/invenio.css +++ b/modules/webstyle/css/invenio.css @@ -2582,22 +2582,19 @@ a:hover.bibMergeImgClickable img { } .youralerts_display_table { - border: 1px solid #bbbbbb; + border: 1px solid #ffcc00; } .youralerts_display_table_header td { - background-color: #7a9bdd; + background-color: #ffffcc; padding: 2px; - border-bottom: 1px solid #3366cc; - border-right: 1px solid #3366cc; + border-bottom: 1px solid #ffcc00; } .youralerts_display_table_footer td { - background-color: #7a9bdd; - color: white; + background-color: #ffffcc; + color: black; padding: 2px; font-size: 80%; text-align: center; - border-bottom: 1px solid #3366cc; - border-right: 1px solid #3366cc; white-space: nowrap; } .youralerts_display_table_footer img { @@ -2618,16 +2615,18 @@ a:hover.bibMergeImgClickable img { vertical-align: bottom; } .youralerts_display_table_content { - background-color: #f2f2fa; + background-color: white; padding: 5px; text-align: left; vertical-align: top; + border-bottom: 1px solid #ffcc00; } .youralerts_display_table_content_mouseover { - background-color: #ebebfa; + background-color: #ffffe3; padding: 5px; text-align: left; vertical-align: top; + border-bottom: 1px solid #ffcc00; } .youralerts_display_table_content_container_left { float: left; @@ -2641,12 +2640,13 @@ a:hover.bibMergeImgClickable img { clear: both; } .youralerts_display_table_counter { - background-color: #f2f2fa; + background-color: white; padding: 5px 2px; text-align: right; vertical-align: top; font-size: 80%; color: gray; + border-bottom: 1px solid #ffcc00; } .youralerts_display_table_content_name { margin-bottom: 5px; @@ -2698,28 +2698,25 @@ a:hover.bibMergeImgClickable img { /* WebSearch module - YourSearches */ .websearch_yoursearches_table { - border: 1px solid #bbbbbb; + border: 1px solid #ffcc00; } .websearch_yoursearches_table_header td { - background-color: #7a9bdd; - color: white; + background-color: #ffffcc; + color: black; padding: 2px; font-weight: bold; font-size: 75%; text-transform: uppercase; - border-bottom: 1px solid #3366cc; - border-right: 1px solid #3366cc; + border-bottom: 1px solid #ffcc00; white-space: nowrap; vertical-align: top; } .websearch_yoursearches_table_footer td { - background-color: #7a9bdd; - color: white; + background-color: #ffffcc; + color: black; padding: 5px; font-size: 80%; text-align: center; - border-bottom: 1px solid #3366cc; - border-right: 1px solid #3366cc; white-space: nowrap; } .websearch_yoursearches_table_footer img { @@ -2740,24 +2737,27 @@ a:hover.bibMergeImgClickable img { vertical-align: bottom; } .websearch_yoursearches_table_content { - background-color: #f2f2fa; + background-color: white; padding: 5px; text-align: left; vertical-align: top; + border-bottom: 1px solid #ffcc00; } .websearch_yoursearches_table_content_mouseover { - background-color: #ebebfa; + background-color: #ffffe3; padding: 5px; text-align: left; vertical-align: top; + border-bottom: 1px solid #ffcc00; } .websearch_yoursearches_table_counter { - background-color: #f2f2fa; + background-color: white; padding: 5px 2px; text-align: right; vertical-align: top; font-size: 80%; color: grey; + border-bottom: 1px solid #ffcc00; } .websearch_yoursearches_table_content_options { margin: 5px; diff --git a/modules/webstyle/img/youralerts_alert_dates.png b/modules/webstyle/img/youralerts_alert_dates.png deleted file mode 100644 index b52ed5bb71c6937ab451a571eb6d476aae1e34bd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 661 zcmV;G0&4wZ4IEgqMPCUtbb!g7PXnVs8FVAy7pZ5cQkpLuT zXJ-OT0lWwyIB+wi+G@4zcDud+NYm7jkr8^m9wP(RCGZM3p<1nm5Q3^ek9NE5;^LyJ zQ4|Hn$H!G&UtbqE%PJjItJR>|>2y>b8ym~pxKz#ZZfS$!L6;Wp+lDN9MnrDjR z*vZMsp&iub=H`r|DAa1TzbcVSIfSo3Ix{ol?(VLQM#E;aX{A!JSS$*pz`;KQ3x$Gl z9Q*L_;L6I1s!5WVB#EkPYir)#-kKyyCiJ&lf$8aKRZFGP(4ZF=7uM_bAzQgzHceA! zXJ-XI0>q6*!zhaKC1hAyT5@@L*{8>HeSK}G(-8;&SCvX7C!}fW#>R%*+uP4B>~Vp^ z!$X1J0MkHve0&`4@9&=lJ^kO^-Fbd~9tKLheZ0uiiHQkMPftT8dK5*r+ig|5-LCa| vJq+~y0leftcmsR_egO_%1NZ{GdrJNT%MwrGizb5800000NkvXXu0mjf6N4YE diff --git a/modules/webstyle/img/youralerts_alert_search.png b/modules/webstyle/img/youralerts_alert_search.png deleted file mode 100644 index 742d87d0eea3ba912475ce8cca9909ff3ccdf13f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 803 zcmV+;1Kj+HP)o%n$G|L0-#AmK~y-)osVlsQ*juFfB$oiZo0YcY|5tQWois;n1Q57T4GX$ z%m}>X1#Id>G?bM5qS2Qikj$VP5>iw=Y4oz@H`JM-MWM0umzSh$R4U{7OgrQ=&40fK-bt zSy%Nrx_`~IV_9*Y%tB;DDM>;I3$sx1sX!@+92_=&UY9iJPvumU>2m?d=Egetl-ZH}6@qrHLSnb9NPd3rKHt>%Y^xBdt*Q)MjgG(A!GxLIe*QKAMF>SG za6HeuJUm>BOFk`XPtlp8ymSDLjRBfgDC610LkAqJ%B*2UB!(3sN+v>>SV&YN0i=N} z_hD|)+H@o_=4D_1=*9q}mP@}%mS*j7v7w?+C2N!VFc8o*oE!&>o8!OCE%+r%Txr!6 z4Tm>-fgYcju#fchbnl}aA!_10?eDY;Wt@Y-!nDTE{60I0vKDHi)!{5F|6w#5O@OZ( z;ILB^rjDYiU%kbASG{apF2o-E55@6Fo$sn#Sb44IpVSAAWoZ#j3002ovPDHLkV1msIXo&y- diff --git a/modules/webstyle/img/yoursearches_search_last_performed.png b/modules/webstyle/img/yoursearches_search_last_performed.png deleted file mode 100644 index b52ed5bb71c6937ab451a571eb6d476aae1e34bd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 661 zcmV;G0&4wZ4IEgqMPCUtbb!g7PXnVs8FVAy7pZ5cQkpLuT zXJ-OT0lWwyIB+wi+G@4zcDud+NYm7jkr8^m9wP(RCGZM3p<1nm5Q3^ek9NE5;^LyJ zQ4|Hn$H!G&UtbqE%PJjItJR>|>2y>b8ym~pxKz#ZZfS$!L6;Wp+lDN9MnrDjR z*vZMsp&iub=H`r|DAa1TzbcVSIfSo3Ix{ol?(VLQM#E;aX{A!JSS$*pz`;KQ3x$Gl z9Q*L_;L6I1s!5WVB#EkPYir)#-kKyyCiJ&lf$8aKRZFGP(4ZF=7uM_bAzQgzHceA! zXJ-XI0>q6*!zhaKC1hAyT5@@L*{8>HeSK}G(-8;&SCvX7C!}fW#>R%*+uP4B>~Vp^ z!$X1J0MkHve0&`4@9&=lJ^kO^-Fbd~9tKLheZ0uiiHQkMPftT8dK5*r+ig|5-LCa| vJq+~y0leftcmsR_egO_%1NZ{GdrJNT%MwrGizb5800000NkvXXu0mjf6N4YE From e7b8434f361d63d19a07ac21552405c52b18d9da Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Tue, 30 Aug 2011 16:34:56 +0200 Subject: [PATCH 21/59] WebSearch: more CSS fallback to default * Changes the arrows used for page navigation in the Your Searches page to match Invenio's default arrows. * Deletes the respective previously added images files. (closes #884) --- modules/websearch/lib/websearch_templates.py | 12 ++++++------ modules/webstyle/img/yoursearches_first_page.png | Bin 1568 -> 0 bytes modules/webstyle/img/yoursearches_last_page.png | Bin 1548 -> 0 bytes modules/webstyle/img/yoursearches_next_page.png | Bin 797 -> 0 bytes .../webstyle/img/yoursearches_previous_page.png | Bin 805 -> 0 bytes 5 files changed, 6 insertions(+), 6 deletions(-) delete mode 100644 modules/webstyle/img/yoursearches_first_page.png delete mode 100644 modules/webstyle/img/yoursearches_last_page.png delete mode 100644 modules/webstyle/img/yoursearches_next_page.png delete mode 100644 modules/webstyle/img/yoursearches_previous_page.png diff --git a/modules/websearch/lib/websearch_templates.py b/modules/websearch/lib/websearch_templates.py index e5dca4571b..b077d184bc 100644 --- a/modules/websearch/lib/websearch_templates.py +++ b/modules/websearch/lib/websearch_templates.py @@ -5198,9 +5198,9 @@ def tmpl_yoursearches_display(self, (CFG_SITE_SECURE_URL, ln, search_query_id, CFG_SITE_URL, _('Edit your existing alert(s)')) + \ '   ' + \ """%s""" % \ - (CFG_SITE_SECURE_URL, ln, search_query_id, CFG_SITE_URL, _('Set up a new alert')) or \ + (CFG_SITE_SECURE_URL, ln, search_query_id, CFG_SITE_URL, _('Set up as a new alert')) or \ """%s""" % \ - (CFG_SITE_SECURE_URL, ln, search_query_id, CFG_SITE_URL, _('Set up a new alert')) + (CFG_SITE_SECURE_URL, ln, search_query_id, CFG_SITE_URL, _('Set up as a new alert')) search_query_options = "%s   %s" % \ (search_query_options_search, \ @@ -5224,10 +5224,10 @@ def tmpl_yoursearches_display(self, footer = '' if paging_navigation[0]: footer += """""" % \ - (CFG_SITE_SECURE_URL, 1, step, cgi.escape(p), ln, '/img/yoursearches_first_page.png') + (CFG_SITE_SECURE_URL, 1, step, cgi.escape(p), ln, '/img/sb.gif') if paging_navigation[1]: footer += """""" % \ - (CFG_SITE_SECURE_URL, page - 1, step, cgi.escape(p), ln, '/img/yoursearches_previous_page.png') + (CFG_SITE_SECURE_URL, page - 1, step, cgi.escape(p), ln, '/img/sp.gif') footer += " " displayed_searches_from = ((page - 1) * step) + 1 displayed_searches_to = paging_navigation[2] and (page * step) or nb_queries_distinct @@ -5236,10 +5236,10 @@ def tmpl_yoursearches_display(self, footer += " " if paging_navigation[2]: footer += """""" % \ - (CFG_SITE_SECURE_URL, page + 1, step, cgi.escape(p), ln, '/img/yoursearches_next_page.png') + (CFG_SITE_SECURE_URL, page + 1, step, cgi.escape(p), ln, '/img/sn.gif') if paging_navigation[3]: footer += """""" % \ - (CFG_SITE_SECURE_URL, paging_navigation[3], step, cgi.escape(p), ln, '/img/yoursearches_last_page.png') + (CFG_SITE_SECURE_URL, paging_navigation[3], step, cgi.escape(p), ln, '/img/se.gif') out += """
    diff --git a/modules/webstyle/img/yoursearches_first_page.png b/modules/webstyle/img/yoursearches_first_page.png deleted file mode 100644 index a16d039aa93a3513f8d2116cf15d017f3cbe85ad..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1568 zcmV+*2H*LKP)QILcfP+W1LkXEA-gw{}sh(r-lt5%H27Em;jAh-~gYH7iUG*r6pLpz=2 zef}twLWt+r&G~Zg{c_IvKE%Yv!~+25JO%*x)T!>o*7g>9Y{}w1z1|)|Qc~hQxDHE< z#mOB``++`hFSWL{XftPel8~9{n&ZyNx{{OQ&S~#x)v~g)5;8JeDV^ONl$w?@m*4-u zyg;@)gHD_{dBQ&!V5Oy{gTMWD>RfkMx3+N6!+toNafP@6{fM&;E$vMN0NZzLr}mCk z0)S0r8>y|mg;<}rmjb~67-Qg!BNz;V>o^z~_@I9fTQ*m+k3QV#A<{5PiDuQGeK8pz zATe>mea$WZP_{es%837%JIDQ$&o@9eyKMphsHzG8NJ>sx+}zwm-a8&R4yh$$z5JdwdNk(2?9=%>#maucz`Q)YQ}v5fK1BT!?en<4Q```KqcurI8mDB`W&Gmsa4yBPsi~plw!y`{u&l%*a~dlvGwDCF1|V0K`slrG8oZYKb=?;m&zHOp-LGGagY&BpeP0G))TufXCyx>7&d{*Q2YR zUU})l#rj}Rcju+Od%w9WD>EDM&iK&n8cRz}xwmNHLl@4TJrnBb?P@-9NpR zZEY>)7cMk5G+rzzDq6U#sHiA5Dk=(ozn=gwB7p0300^=yYl1Ax8i*hoi~s%Cv+Ra92|T66Aq#9FLO6h$%`Ob9bcFw5bH2#-WWcoYOd zfNCAB33j`UL{UT_7=S1mV6~3cw!O3M7=ZD`Yt~`?hBqe~jpEBrNBok)2j=72fR~_w zan2!065?!jzWVvqKLN-hB5W$#7_U)qx!qgwvt z3l=O`OGIl={``}4?%e5IizT)qH+RN3lhFjLY9)5Q?vK&6=l@6zX(1vaL`Fsq%(}ba zIAct^HIlWpwU{(%(tmsP@OKAYDM{%UR8{S{-K-GN@a%$!@ZI-^Cy!Drt^WevHBK#s Snf#mp0000 diff --git a/modules/webstyle/img/yoursearches_last_page.png b/modules/webstyle/img/yoursearches_last_page.png deleted file mode 100644 index c61494c785551a39a17c19ea3dcd458e53aeec0a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1548 zcmV+n2J`ueP)-nVx{-B{UR182g5kqCO93Cac<2TJ&u7!`E5P5BbWq7Eo!>pIwM*WT`nj^5te z-uB+U#~&M7NsQ00lPAwP?>WyoFaBu$Z48D`TX+0X*z84%13iAxKhQ^yJwDF_0LZdz z1%Q@T=NR%}_wOaR;l$4C9-v31~eR%qjarfpm z+iXRx$S=76$xD|m3IHf5D45vhb`b!~nmw~?+VrVUfQW820V3kJ;zKnDilAvKeRW_T z0YD_8(a~>e2mn+qOsuJxY2^CnX#rAq2g@>Aj}y72C(^u_0bnNF)-pW$PyQ;wS9BSWKrERk=hYTV zRF6Q{qmU#CAML2NmI(L**S%YHiF`0KUY+} zci#;08V%gMA)HsQQ0Z%>bBTzqUUiaWnkGWZO@IMH2=>6#DTruKH)xS4lyC@)bC3{h z%-y50V#RZH0AQLXxDWy0)@}&DZVjb~hQOL569Nu187JRH5q;>h0^LJ)c!}t}A_cLw}Ew z3ILUrmE(T+@faDVL8fVv9@GEU2M2<5@bCfom33>kIW9ZXhzQG9Eb|f(nF*6xT`jbJ z!>cw)mIK^03}~7`02c7mM+~15#M?0^hve4`iqM>ql*!VXkd(i zF$N;R;-&yWVE+8a?_RKA;YQB+#)}u5btM#%03aHTz=)gh`(-W#0^Q6sO&Dez7A}Ev z27rNqfip(d%#rNOnNz6w`pBQHEp6*Yjmo~Js_ONB|78ZrB7vBo5gD1x?QTQWp4~y` zm6p|!s9FUeR|o+jf}(_(q$tYZ-Utws0LX$&YbLgrZ^MPg3!izt-cM4bbjSI|Qv?7r zADTH4U@%Q%$Q97YtZY`bw+iRZo;`8>ddH@$5xEzR|9Dj7jvCMMCS(Z!(Bn}s280!b zNoqLErIZvTOo8%>a;3KR#A~wOegpN~lv3)0huW6RdfvHJR- zH~M^C-*tI?zQNZ2f!yED#*Qx>oGBpyFsQ1^S$0l#UV7SHV>&xMzXM3%f517<89(8^ yNo_87BLL%ngg7 z&2i^-$G*jWuh&~5+=Fh#Xj}@Lt>sZ7`IU7&$LHG4_*_q22I-yjD&M}RAxOYrFyQ8` zo6-IkuXZYujHsyCymyDC+6h2l5mL(ac=p^?1>m>706_l0G`oif1BJ^Ax7XHK;|t2W zskfX+er0CniPkiC#!;LBcWg+9#=@V%HYw-uv`>?yFpkUc! zpXzfpSoa<+-&8(FM2vqB31mRo zP>3{5Ln@Vo5CTL5W}UM0IzWut+PXFMR_k#=;`qpjXa3IJ+ru&(o*_-spv`BY2nitw z0D(XdM7+FkCHZ_lblvETjg5{US-ontTgKz@pDQzbBOYHChjn4I)bcvX~E*TNJEi{x^R5gQ->95gt(i;9R z4nS_D&C>61wD$Rym%pO0u+RViR;zXIAaAvnH#?k{y&jL7)*`DE6n7#XD=umjOePcA zsx0vR2&pP6EuC#`SDWljc9kMSfvv4A0sxW-9M1y)6pD07(C6yjI9i<*rRGv@b8{Wr zi3GqAqOk}VEpbIf#pn8-KWpgj>GJw~0q8V2NRlOyJUIoaDQQro zXF!>e31vnmWHJG{dOfkh!9ktfZeRHu=6?IVLYYjsnP)U!X+D1u>zgr12?-qIu(-5{ z7ccwIGPm2UEh#DS??*;N_&74GoSU1iRco|QYR}XeBasN+5522re!o9WtJTI1VogL~ jjP2GRMuyDlY_0D%<{TiF_>Qx&00000NkvXXu0mjfk8WAP From b2e235dab56c04c7f7177a34f2535634fc5e30ab Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Tue, 30 Aug 2011 17:25:59 +0200 Subject: [PATCH 22/59] WebAlert: paging for youralerts * Introduces page navigation support for the "Your Alerts" page. (closes #885) --- modules/webalert/lib/webalert.py | 69 ++++++++++++++++--- modules/webalert/lib/webalert_templates.py | 35 ++++++++-- modules/webalert/lib/webalert_webinterface.py | 8 ++- 3 files changed, 96 insertions(+), 16 deletions(-) diff --git a/modules/webalert/lib/webalert.py b/modules/webalert/lib/webalert.py index 67b0366139..1c320cd185 100644 --- a/modules/webalert/lib/webalert.py +++ b/modules/webalert/lib/webalert.py @@ -35,6 +35,8 @@ import invenio.template webalert_templates = invenio.template.load('webalert') +CFG_WEBALERT_YOURALERTS_MAX_NUMBER_OF_DISPLAYED_ALERTS = 20 + ### IMPLEMENTATION class AlertError(Exception): @@ -173,11 +175,13 @@ def perform_add_alert(alert_name, frequency, notification, run_sql(query, params) out = _("The alert %s has been added to your profile.") out %= '' + cgi.escape(alert_name) + '' - out += perform_request_youralerts_display(uid, idq=None, ln=ln) + out += perform_request_youralerts_display(uid, idq=0, ln=ln) return out def perform_request_youralerts_display(uid, - idq=None, + idq=0, + page=1, + step=CFG_WEBALERT_YOURALERTS_MAX_NUMBER_OF_DISPLAYED_ALERTS, ln=CFG_SITE_LANG): """ Display a list of the user defined alerts. If a specific query id is defined @@ -189,6 +193,12 @@ def perform_request_youralerts_display(uid, @param idq: The specified query id for which to display the user alerts @type idq: int + @param page: + @type page: integer + + @param step: + @type step: integer + @param ln: The interface language @type ln: string @@ -203,6 +213,42 @@ def perform_request_youralerts_display(uid, else: idq_clause = "" + query_nb_alerts = """ SELECT COUNT(*) + FROM user_query_basket AS uqb + WHERE uqb.id_user=%%s + %s""" % (idq_clause,) + params_nb_alerts = (uid,) + result_nb_alerts = run_sql(query_nb_alerts, params_nb_alerts) + nb_alerts = result_nb_alerts[0][0] + + # The real page starts counting from 0, i.e. minus 1 from the human page + real_page = page - 1 + # The step needs to be a positive integer + if (step <= 0): + step = CFG_WEBALERT_YOURALERTS_MAX_NUMBER_OF_DISPLAYED_ALERTS + # The maximum real page is the integer division of the total number of + # searches and the searches displayed per page + max_real_page = (nb_alerts / step) - (not (nb_alerts % step) and 1 or 0) + # Check if the selected real page exceeds the maximum real page and reset + # if needed + if (real_page >= max_real_page): + #if ((nb_queries_distinct % step) != 0): + # real_page = max_real_page + #else: + # real_page = max_real_page - 1 + real_page = max_real_page + page = real_page + 1 + elif (real_page < 0): + real_page = 0 + page = 1 + # Calculate the start value for the SQL LIMIT constraint + limit_start = real_page * step + # Calculate the display of the paging navigation arrows for the template + paging_navigation = (real_page >= 2, + real_page >= 1, + real_page <= (max_real_page - 1), + (real_page <= (max_real_page - 2)) and (max_real_page + 1)) + # query the database query = """ SELECT q.id, q.urlargs, @@ -220,10 +266,11 @@ def perform_request_youralerts_display(uid, ON uqb.id_basket=bsk.id WHERE uqb.id_user=%%s %s - ORDER BY uqb.alert_name ASC""" % ('%%Y-%%m-%%d %%H:%%i:%%s', - '%%Y-%%m-%%d %%H:%%i:%%s', - idq_clause,) - params = (uid,) + ORDER BY uqb.alert_name ASC + LIMIT %%s,%%s""" % ('%%Y-%%m-%%d %%H:%%i:%%s', + '%%Y-%%m-%%d %%H:%%i:%%s', + idq_clause,) + params = (uid, limit_start, step) result = run_sql(query, params) alerts = [] @@ -261,8 +308,12 @@ def perform_request_youralerts_display(uid, # link to the "add new alert" form out = webalert_templates.tmpl_youralerts_display(ln=ln, alerts=alerts, - guest=isGuestUser(uid), + nb_alerts=nb_alerts, idq=idq, + page=page, + step=step, + paging_navigation=paging_navigation, + guest=isGuestUser(uid), guesttxt=warning_guest_user(type="alerts", ln=ln)) return out @@ -291,7 +342,7 @@ def perform_remove_alert(alert_name, id_query, id_basket, uid, ln=CFG_SITE_LANG) out += "The alert %s has been removed from your profile.

    \n" % cgi.escape(alert_name) else: out += "Unable to remove alert %s.

    \n" % cgi.escape(alert_name) - out += perform_request_youralerts_display(uid, idq=None, ln=ln) + out += perform_request_youralerts_display(uid, idq=0, ln=ln) return out @@ -351,7 +402,7 @@ def perform_update_alert(alert_name, frequency, notification, id_basket, id_quer run_sql(query, params) out += _("The alert %s has been successfully updated.") % ("" + cgi.escape(alert_name) + "",) - out += "

    \n" + perform_request_youralerts_display(uid, idq=None, ln=ln) + out += "

    \n" + perform_request_youralerts_display(uid, idq=0, ln=ln) return out def is_selected(var, fld): diff --git a/modules/webalert/lib/webalert_templates.py b/modules/webalert/lib/webalert_templates.py index 957be0a36d..aaaa1e35cc 100644 --- a/modules/webalert/lib/webalert_templates.py +++ b/modules/webalert/lib/webalert_templates.py @@ -264,7 +264,11 @@ def tmpl_input_alert(self, ln, query, alert_name, action, frequency, notificatio def tmpl_youralerts_display(self, ln, alerts, + nb_alerts, idq, + page, + step, + paging_navigation, guest, guesttxt): """ @@ -325,14 +329,14 @@ def tmpl_youralerts_display(self, # Diplay a message about the number of alerts. if idq: msg = _('You have defined %(number_of_alerts)s alerts based on that search query.') % \ - {'number_of_alerts': '' + str(len(alerts)) + ''} + {'number_of_alerts': '' + str(nb_alerts) + ''} msg += '
    ' msg += _('You may want to %(new_alert)s or display all %(youralerts)s.') % \ {'new_alert': '%s' % (CFG_SITE_SECURE_URL, ln, idq, _('define a new one')), 'youralerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('your alerts'))} else: msg = _('You have defined a total of %(number_of_alerts)s alerts.') % \ - {'number_of_alerts': '' + str(len(alerts)) + ''} + {'number_of_alerts': '' + str(nb_alerts) + ''} msg += '
    ' msg += _('You may define new alert based on %(yoursearches)s, the %(popular_alerts)s or just by %(search_interface)s.') % \ {'yoursearches': '%s' % (CFG_SITE_SECURE_URL, ln, _('your searches')), @@ -340,7 +344,7 @@ def tmpl_youralerts_display(self, 'search_interface': '%s' %(CFG_SITE_URL, ln, _('searching for something new'))} out = '

    ' + msg + '

    ' - counter = 0 + counter = (page - 1) * step youralerts_display_html = "" for alert in alerts: counter += 1 @@ -433,6 +437,26 @@ def tmpl_youralerts_display(self, 'alert_details_options': alert_details_options, 'alert_details_creation_last_run_dates': alert_details_creation_last_run_dates} + footer = '' + if paging_navigation[0]: + footer += """""" % \ + (CFG_SITE_SECURE_URL, 1, step, idq, ln, '/img/sb.gif') + if paging_navigation[1]: + footer += """""" % \ + (CFG_SITE_SECURE_URL, page - 1, step, idq, ln, '/img/sp.gif') + footer += " " + displayed_alerts_from = ((page - 1) * step) + 1 + displayed_alerts_to = paging_navigation[2] and (page * step) or nb_alerts + footer += _('Displaying alerts %i to %i from %i total alerts') % \ + (displayed_alerts_from, displayed_alerts_to, nb_alerts) + footer += " " + if paging_navigation[2]: + footer += """""" % \ + (CFG_SITE_SECURE_URL, page + 1, step, idq, ln, '/img/sn.gif') + if paging_navigation[3]: + footer += """""" % \ + (CFG_SITE_SECURE_URL, paging_navigation[3], step, idq, ln, '/img/se.gif') + out += """
    @@ -442,13 +466,14 @@ def tmpl_youralerts_display(self, - + %(youralerts_display_html)s -
    %(footer)s
    """ % {'youralerts_display_html': youralerts_display_html} +""" % {'footer': footer, + 'youralerts_display_html': youralerts_display_html} return out diff --git a/modules/webalert/lib/webalert_webinterface.py b/modules/webalert/lib/webalert_webinterface.py index a189e9c7fd..419f41b0f9 100644 --- a/modules/webalert/lib/webalert_webinterface.py +++ b/modules/webalert/lib/webalert_webinterface.py @@ -239,8 +239,10 @@ def modify(self, req, form): def display(self, req, form): - argd = wash_urlargd(form, {'idq': (int, None), - }) + argd = wash_urlargd(form, {'idq': (int, 0), + 'page': (int, 1), + 'step': (int, 20), + 'ln': (str, '')}) uid = getUid(req) @@ -277,6 +279,8 @@ def display(self, req, form): return page(title=_("Your Alerts"), body=perform_request_youralerts_display(uid, idq=argd['idq'], + page=argd['page'], + step=argd['step'], ln=argd['ln']), navtrail= """%(account)s""" % \ {'sitesecureurl' : CFG_SITE_SECURE_URL, From 5d21b1738eb8affda864ef059fb7a91c44f8b378 Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Thu, 1 Sep 2011 18:10:42 +0200 Subject: [PATCH 23/59] WebAlert: Searching for youralerts * Introduces searching in the user's defined alerts based on the alerts' names and arguments. * Fixes small bug in the paging navigation that caused a crash when no alerts were to be displayed. * Adds user confirm step before permanently deleting an alert. (closes #886) --- modules/webalert/lib/webalert.py | 201 ++++++++++-------- modules/webalert/lib/webalert_templates.py | 81 ++++++- modules/webalert/lib/webalert_webinterface.py | 2 + 3 files changed, 191 insertions(+), 93 deletions(-) diff --git a/modules/webalert/lib/webalert.py b/modules/webalert/lib/webalert.py index 1c320cd185..e0a3d07355 100644 --- a/modules/webalert/lib/webalert.py +++ b/modules/webalert/lib/webalert.py @@ -21,6 +21,7 @@ import cgi import time +from urllib import quote from invenio.config import CFG_SITE_LANG from invenio.dbquery import run_sql @@ -182,6 +183,7 @@ def perform_request_youralerts_display(uid, idq=0, page=1, step=CFG_WEBALERT_YOURALERTS_MAX_NUMBER_OF_DISPLAYED_ALERTS, + p='', ln=CFG_SITE_LANG): """ Display a list of the user defined alerts. If a specific query id is defined @@ -209,110 +211,139 @@ def perform_request_youralerts_display(uid, out = "" if idq: - idq_clause = "AND uqb.id_query=%i" % (idq,) + idq_clause = "q.id=%i" % (idq,) else: idq_clause = "" - query_nb_alerts = """ SELECT COUNT(*) + search_clause = "" + search_clause_urlargs = [] + search_clause_alert_name = [] + if p: + p_stripped = p.strip() + p_stripped_args = p.split() + sql_p_stripped_args = ['\'%%' + quote(p_stripped_arg).replace('%','%%') + '%%\'' for p_stripped_arg in p_stripped_args] + for sql_p_stripped_arg in sql_p_stripped_args: + search_clause_urlargs.append("q.urlargs LIKE %s" % (sql_p_stripped_arg,)) + search_clause_alert_name.append("uqb.alert_name LIKE %s" % (sql_p_stripped_arg,)) + search_clause = "((%s) OR (%s))" % (" AND ".join(search_clause_urlargs), + " AND ".join(search_clause_alert_name)) + + idq_and_search_clause_list = [clause for clause in (idq_clause, search_clause) if clause] + idq_and_search_clause = ' AND '.join(idq_and_search_clause_list) + + query_nb_alerts = """ SELECT COUNT(IF((uqb.id_user=%%s + %s),uqb.id_query,NULL)), + COUNT(q.id) FROM user_query_basket AS uqb - WHERE uqb.id_user=%%s - %s""" % (idq_clause,) + RIGHT JOIN query AS q + ON uqb.id_query=q.id + %s""" % ((search_clause and ' AND ' + search_clause), + (idq_clause and ' WHERE ' + idq_clause)) params_nb_alerts = (uid,) result_nb_alerts = run_sql(query_nb_alerts, params_nb_alerts) nb_alerts = result_nb_alerts[0][0] - - # The real page starts counting from 0, i.e. minus 1 from the human page - real_page = page - 1 - # The step needs to be a positive integer - if (step <= 0): - step = CFG_WEBALERT_YOURALERTS_MAX_NUMBER_OF_DISPLAYED_ALERTS - # The maximum real page is the integer division of the total number of - # searches and the searches displayed per page - max_real_page = (nb_alerts / step) - (not (nb_alerts % step) and 1 or 0) - # Check if the selected real page exceeds the maximum real page and reset - # if needed - if (real_page >= max_real_page): - #if ((nb_queries_distinct % step) != 0): - # real_page = max_real_page - #else: - # real_page = max_real_page - 1 - real_page = max_real_page - page = real_page + 1 - elif (real_page < 0): - real_page = 0 - page = 1 - # Calculate the start value for the SQL LIMIT constraint - limit_start = real_page * step - # Calculate the display of the paging navigation arrows for the template - paging_navigation = (real_page >= 2, - real_page >= 1, - real_page <= (max_real_page - 1), - (real_page <= (max_real_page - 2)) and (max_real_page + 1)) - - # query the database - query = """ SELECT q.id, - q.urlargs, - uqb.id_basket, - bsk.name, - uqb.alert_name, - uqb.frequency, - uqb.notification, - DATE_FORMAT(uqb.date_creation,'%s'), - DATE_FORMAT(uqb.date_lastrun,'%s') - FROM user_query_basket uqb - LEFT JOIN query q - ON uqb.id_query=q.id - LEFT JOIN bskBASKET bsk - ON uqb.id_basket=bsk.id - WHERE uqb.id_user=%%s - %s - ORDER BY uqb.alert_name ASC - LIMIT %%s,%%s""" % ('%%Y-%%m-%%d %%H:%%i:%%s', - '%%Y-%%m-%%d %%H:%%i:%%s', - idq_clause,) - params = (uid, limit_start, step) - result = run_sql(query, params) - - alerts = [] - for (query_id, - query_args, - bsk_id, - bsk_name, - alert_name, - alert_frequency, - alert_notification, - alert_creation, - alert_last_run) in result: - try: - if not query_id: - raise StandardError("""\ + nb_queries = result_nb_alerts[0][1] + + # In case we do have some alerts, proceed with the needed calculations and + # fetching them from the database + if nb_alerts: + # The real page starts counting from 0, i.e. minus 1 from the human page + real_page = page - 1 + # The step needs to be a positive integer + if (step <= 0): + step = CFG_WEBALERT_YOURALERTS_MAX_NUMBER_OF_DISPLAYED_ALERTS + # The maximum real page is the integer division of the total number of + # searches and the searches displayed per page + max_real_page = nb_alerts and ((nb_alerts / step) - (not (nb_alerts % step) and 1 or 0)) or 0 + # Check if the selected real page exceeds the maximum real page and reset + # if needed + if (real_page >= max_real_page): + #if ((nb_queries_distinct % step) != 0): + # real_page = max_real_page + #else: + # real_page = max_real_page - 1 + real_page = max_real_page + page = real_page + 1 + elif (real_page < 0): + real_page = 0 + page = 1 + # Calculate the start value for the SQL LIMIT constraint + limit_start = real_page * step + # Calculate the display of the paging navigation arrows for the template + paging_navigation = (real_page >= 2, + real_page >= 1, + real_page <= (max_real_page - 1), + (real_page <= (max_real_page - 2)) and (max_real_page + 1)) + + query = """ SELECT q.id, + q.urlargs, + uqb.id_basket, + bsk.name, + uqb.alert_name, + uqb.frequency, + uqb.notification, + DATE_FORMAT(uqb.date_creation,'%s'), + DATE_FORMAT(uqb.date_lastrun,'%s') + FROM user_query_basket uqb + LEFT JOIN query q + ON uqb.id_query=q.id + LEFT JOIN bskBASKET bsk + ON uqb.id_basket=bsk.id + WHERE uqb.id_user=%%s + %s + %s + ORDER BY uqb.alert_name ASC + LIMIT %%s,%%s""" % ('%%Y-%%m-%%d %%H:%%i:%%s', + '%%Y-%%m-%%d %%H:%%i:%%s', + (idq_clause and ' AND ' + idq_clause), + (search_clause and ' AND ' + search_clause)) + params = (uid, limit_start, step) + result = run_sql(query, params) + + alerts = [] + for (query_id, + query_args, + bsk_id, + bsk_name, + alert_name, + alert_frequency, + alert_notification, + alert_creation, + alert_last_run) in result: + try: + if not query_id: + raise StandardError("""\ Warning: I have detected a bad alert for user id %d. It seems one of his/her alert queries was deleted from the 'query' table. Please check this and delete it if needed. Otherwise no problem, I'm continuing with the other alerts now. Here are all the alerts defined by this user: %s""" % (uid, repr(result))) - alerts.append({'queryid' : query_id, - 'queryargs' : query_args, - 'textargs' : get_textual_query_info_from_urlargs(query_args, ln=ln), - 'userid' : uid, - 'basketid' : bsk_id, - 'basketname' : bsk_name, - 'alertname' : alert_name, - 'frequency' : alert_frequency, - 'notification' : alert_notification, - 'created' : alert_creation, - 'lastrun' : alert_last_run}) - except StandardError: - register_exception(alert_admin=True) - - # link to the "add new alert" form + alerts.append({'queryid' : query_id, + 'queryargs' : query_args, + 'textargs' : get_textual_query_info_from_urlargs(query_args, ln=ln), + 'userid' : uid, + 'basketid' : bsk_id, + 'basketname' : bsk_name, + 'alertname' : alert_name, + 'frequency' : alert_frequency, + 'notification' : alert_notification, + 'created' : alert_creation, + 'lastrun' : alert_last_run}) + except StandardError: + register_exception(alert_admin=True) + else: + alerts = [] + paging_navigation = () + out = webalert_templates.tmpl_youralerts_display(ln=ln, alerts=alerts, nb_alerts=nb_alerts, + nb_queries=nb_queries, idq=idq, page=page, step=step, paging_navigation=paging_navigation, + p=p, guest=isGuestUser(uid), guesttxt=warning_guest_user(type="alerts", ln=ln)) return out diff --git a/modules/webalert/lib/webalert_templates.py b/modules/webalert/lib/webalert_templates.py index aaaa1e35cc..2322b340b2 100644 --- a/modules/webalert/lib/webalert_templates.py +++ b/modules/webalert/lib/webalert_templates.py @@ -265,10 +265,12 @@ def tmpl_youralerts_display(self, ln, alerts, nb_alerts, + nb_queries, idq, page, step, paging_navigation, + p, guest, guesttxt): """ @@ -309,13 +311,44 @@ def tmpl_youralerts_display(self, # In case the user has not yet defined any alerts display only the # following message - if not alerts: - if idq: - msg = _('You have not defined any alerts yet based on that search query.') + if not nb_alerts: + if idq and not p: + if nb_queries: + msg = _('You have not defined any alerts yet based on that search query.') + msg += "
    " + msg += _('You may want to %(new_alert)s or display all %(youralerts)s.') % \ + {'new_alert': '%s' % (CFG_SITE_SECURE_URL, ln, idq, _('define one now')), + 'youralerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('your alerts'))} + else: + msg = _('The selected search query seems to be invalid.') + msg += "
    " + msg += _('You may define new alert based on %(yoursearches)s, the %(popular_alerts)s or just by %(search_interface)s.') % \ + {'yoursearches': '%s' % (CFG_SITE_SECURE_URL, ln, _('your searches')), + 'popular_alerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('popular alerts')), + 'search_interface': '%s' %(CFG_SITE_URL, ln, _('searching for something new'))} + elif p and not idq: + msg = _('You have not defined any alerts yet including the terms %s.') % \ + ('' + cgi.escape(p) + '',) msg += "
    " - msg += _('You may want to %(new_alert)s or display all %(youralerts)s.') % \ - {'new_alert': '%s' % (CFG_SITE_SECURE_URL, ln, idq, _('define one now')), - 'youralerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('your alerts'))} + msg += _('You may define new alert based on %(yoursearches)s, the %(popular_alerts)s or just by %(search_interface)s.') % \ + {'yoursearches': '%s' % (CFG_SITE_SECURE_URL, ln, _('your searches')), + 'popular_alerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('popular alerts')), + 'search_interface': '%s' %(CFG_SITE_URL, ln, _('searching for something new'))} + elif p and idq: + if nb_queries: + msg = _('You have not defined any alerts yet based on that search query including the terms %s.') % \ + ('' + cgi.escape(p) + '',) + msg += "
    " + msg += _('You may want to %(new_alert)s or display all %(youralerts)s.') % \ + {'new_alert': '%s' % (CFG_SITE_SECURE_URL, ln, idq, _('define one now')), + 'youralerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('your alerts'))} + else: + msg = _('The selected search query seems to be invalid.') + msg += "
    " + msg += _('You may define new alert based on %(yoursearches)s, the %(popular_alerts)s or just by %(search_interface)s.') % \ + {'yoursearches': '%s' % (CFG_SITE_SECURE_URL, ln, _('your searches')), + 'popular_alerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('popular alerts')), + 'search_interface': '%s' %(CFG_SITE_URL, ln, _('searching for something new'))} else: msg = _('You have not defined any alerts yet.') msg += '
    ' @@ -327,13 +360,30 @@ def tmpl_youralerts_display(self, return out # Diplay a message about the number of alerts. - if idq: + if idq and not p: msg = _('You have defined %(number_of_alerts)s alerts based on that search query.') % \ {'number_of_alerts': '' + str(nb_alerts) + ''} msg += '
    ' msg += _('You may want to %(new_alert)s or display all %(youralerts)s.') % \ {'new_alert': '%s' % (CFG_SITE_SECURE_URL, ln, idq, _('define a new one')), 'youralerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('your alerts'))} + elif p and not idq: + msg = _('You have defined %(number_of_alerts)s alerts including the terms %(p)s.') % \ + {'p': '' + cgi.escape(p) + '', + 'number_of_alerts': '' + str(nb_alerts) + ''} + msg += '
    ' + msg += _('You may define new alert based on %(yoursearches)s, the %(popular_alerts)s or just by %(search_interface)s.') % \ + {'yoursearches': '%s' % (CFG_SITE_SECURE_URL, ln, _('your searches')), + 'popular_alerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('popular alerts')), + 'search_interface': '%s' %(CFG_SITE_URL, ln, _('searching for something new'))} + elif idq and p: + msg = _('You have defined %(number_of_alerts)s alerts based on that search query including the terms %(p)s.') % \ + {'p': '' + cgi.escape(p) + '', + 'number_of_alerts': '' + str(nb_alerts) + ''} + msg += '
    ' + msg += _('You may want to %(new_alert)s or display all %(youralerts)s.') % \ + {'new_alert': '%s' % (CFG_SITE_SECURE_URL, ln, idq, _('define a new one')), + 'youralerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('your alerts'))} else: msg = _('You have defined a total of %(number_of_alerts)s alerts.') % \ {'number_of_alerts': '' + str(nb_alerts) + ''} @@ -344,6 +394,19 @@ def tmpl_youralerts_display(self, 'search_interface': '%s' %(CFG_SITE_URL, ln, _('searching for something new'))} out = '

    ' + msg + '

    ' + # Search form + search_form = """ +
    + %(search_text)s + + +
    + """ % {'search_text': _('Search all your alerts for'), + 'action': '%s/youralerts/display?ln=%s' % (CFG_SITE_SECURE_URL, ln), + 'p': cgi.escape(p), + 'submit_label': _('Search')} + out += '

    ' + search_form + '

    ' + counter = (page - 1) * step youralerts_display_html = "" for alert in alerts: @@ -406,7 +469,9 @@ def tmpl_youralerts_display(self, 'idq' : alert_query_id, 'name' : alert_name, 'idb' : alert_basket_id}, - _('Delete')) + _('Delete'), + {'onclick': 'return confirm(\'%s\')' % \ + (_('Are you sure you want to permanently delete this alert?'),)}) alert_details_options = '' % (CFG_SITE_URL,) + \ alert_details_options_edit + \ '   ' + \ diff --git a/modules/webalert/lib/webalert_webinterface.py b/modules/webalert/lib/webalert_webinterface.py index 419f41b0f9..6c2c850d99 100644 --- a/modules/webalert/lib/webalert_webinterface.py +++ b/modules/webalert/lib/webalert_webinterface.py @@ -242,6 +242,7 @@ def display(self, req, form): argd = wash_urlargd(form, {'idq': (int, 0), 'page': (int, 1), 'step': (int, 20), + 'p': (str, ''), 'ln': (str, '')}) uid = getUid(req) @@ -281,6 +282,7 @@ def display(self, req, form): idq=argd['idq'], page=argd['page'], step=argd['step'], + p=argd['p'], ln=argd['ln']), navtrail= """%(account)s""" % \ {'sitesecureurl' : CFG_SITE_SECURE_URL, From acfd28dc83911118b0f4529432763bca7c27b26d Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Thu, 1 Sep 2011 18:26:28 +0200 Subject: [PATCH 24/59] WebSearch: Fix paging bug in yoursearches * Fixes small bug in the paging navigation that caused a crash when no searches were to be displayed. * Introduces fix that skips querying the database again when there are no matching user searches. (closes #887) --- modules/webalert/lib/webalert.py | 2 +- modules/websearch/lib/websearch_templates.py | 2 +- .../websearch/lib/websearch_yoursearches.py | 111 +++++++++--------- 3 files changed, 59 insertions(+), 56 deletions(-) diff --git a/modules/webalert/lib/webalert.py b/modules/webalert/lib/webalert.py index e0a3d07355..936c7c355d 100644 --- a/modules/webalert/lib/webalert.py +++ b/modules/webalert/lib/webalert.py @@ -254,7 +254,7 @@ def perform_request_youralerts_display(uid, step = CFG_WEBALERT_YOURALERTS_MAX_NUMBER_OF_DISPLAYED_ALERTS # The maximum real page is the integer division of the total number of # searches and the searches displayed per page - max_real_page = nb_alerts and ((nb_alerts / step) - (not (nb_alerts % step) and 1 or 0)) or 0 + max_real_page = nb_alerts and ((nb_alerts / step) - (not (nb_alerts % step) and 1 or 0)) # Check if the selected real page exceeds the maximum real page and reset # if needed if (real_page >= max_real_page): diff --git a/modules/websearch/lib/websearch_templates.py b/modules/websearch/lib/websearch_templates.py index b077d184bc..46e65902a7 100644 --- a/modules/websearch/lib/websearch_templates.py +++ b/modules/websearch/lib/websearch_templates.py @@ -5171,7 +5171,7 @@ def tmpl_yoursearches_display(self, - """ % {'search_text': _('Search inside your searches for'), + """ % {'search_text': _('Search inside all your searches for'), 'action': '%s/yoursearches/display?ln=%s' % (CFG_SITE_SECURE_URL, ln), 'p': cgi.escape(p), 'submit_label': _('Search')} diff --git a/modules/websearch/lib/websearch_yoursearches.py b/modules/websearch/lib/websearch_yoursearches.py index b56a9f22f3..23e9c549d4 100644 --- a/modules/websearch/lib/websearch_yoursearches.py +++ b/modules/websearch/lib/websearch_yoursearches.py @@ -83,60 +83,63 @@ def perform_request_yoursearches_display(uid, nb_queries_total = res_nb_queries[0][0] nb_queries_distinct = res_nb_queries[0][1] - # The real page starts counting from 0, i.e. minus 1 from the human page - real_page = page - 1 - # The step needs to be a positive integer - if (step <= 0): - step = CFG_WEBSEARCH_YOURSEARCHES_MAX_NUMBER_OF_DISPLAYED_SEARCHES - # The maximum real page is the integer division of the total number of - # searches and the searches displayed per page - max_real_page = (nb_queries_distinct / step) - (not (nb_queries_distinct % step) and 1 or 0) - # Check if the selected real page exceeds the maximum real page and reset - # if needed - if (real_page >= max_real_page): - #if ((nb_queries_distinct % step) != 0): - # real_page = max_real_page - #else: - # real_page = max_real_page - 1 - real_page = max_real_page - page = real_page + 1 - elif (real_page < 0): - real_page = 0 - page = 1 - # Calculate the start value for the SQL LIMIT constraint - limit_start = real_page * step - # Calculate the display of the paging navigation arrows for the template - paging_navigation = (real_page >= 2, - real_page >= 1, - real_page <= (max_real_page - 1), - (real_page <= (max_real_page - 2)) and (max_real_page + 1)) - - - # Calculate the user search queries - query = """ SELECT DISTINCT(q.id), - q.urlargs, - DATE_FORMAT(MAX(uq.date),'%s') - FROM query q, - user_query uq - WHERE uq.id_user=%%s - AND uq.id_query=q.id - %s - GROUP BY uq.id_query - ORDER BY MAX(uq.date) DESC - LIMIT %%s,%%s""" % ('%%Y-%%m-%%d %%H:%%i:%%s', search_clause,) - params = (uid, limit_start, step) - result = run_sql(query, params) - - search_queries = [] - if result: - for search_query in result: - search_query_id = search_query[0] - search_query_args = search_query[1] - search_query_lastrun = search_query[2] or _("unknown") - search_queries.append({'id' : search_query_id, - 'args' : search_query_args, - 'lastrun' : search_query_lastrun, - 'user_alerts' : count_user_alerts_for_given_query(uid, search_query_id)}) + if nb_queries_total: + # The real page starts counting from 0, i.e. minus 1 from the human page + real_page = page - 1 + # The step needs to be a positive integer + if (step <= 0): + step = CFG_WEBSEARCH_YOURSEARCHES_MAX_NUMBER_OF_DISPLAYED_SEARCHES + # The maximum real page is the integer division of the total number of + # searches and the searches displayed per page + max_real_page = nb_queries_distinct and ((nb_queries_distinct / step) - (not (nb_queries_distinct % step) and 1 or 0)) + # Check if the selected real page exceeds the maximum real page and reset + # if needed + if (real_page >= max_real_page): + #if ((nb_queries_distinct % step) != 0): + # real_page = max_real_page + #else: + # real_page = max_real_page - 1 + real_page = max_real_page + page = real_page + 1 + elif (real_page < 0): + real_page = 0 + page = 1 + # Calculate the start value for the SQL LIMIT constraint + limit_start = real_page * step + # Calculate the display of the paging navigation arrows for the template + paging_navigation = (real_page >= 2, + real_page >= 1, + real_page <= (max_real_page - 1), + (real_page <= (max_real_page - 2)) and (max_real_page + 1)) + + # Calculate the user search queries + query = """ SELECT DISTINCT(q.id), + q.urlargs, + DATE_FORMAT(MAX(uq.date),'%s') + FROM query q, + user_query uq + WHERE uq.id_user=%%s + AND uq.id_query=q.id + %s + GROUP BY uq.id_query + ORDER BY MAX(uq.date) DESC + LIMIT %%s,%%s""" % ('%%Y-%%m-%%d %%H:%%i:%%s', search_clause,) + params = (uid, limit_start, step) + result = run_sql(query, params) + + search_queries = [] + if result: + for search_query in result: + search_query_id = search_query[0] + search_query_args = search_query[1] + search_query_lastrun = search_query[2] or _("unknown") + search_queries.append({'id' : search_query_id, + 'args' : search_query_args, + 'lastrun' : search_query_lastrun, + 'user_alerts' : count_user_alerts_for_given_query(uid, search_query_id)}) + else: + search_queries = [] + paging_navigation = () return websearch_templates.tmpl_yoursearches_display( nb_queries_total = nb_queries_total, From b8fd5260b4304e1697a9f8c057b0b08a92627938 Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Thu, 2 Feb 2012 15:36:30 +0100 Subject: [PATCH 25/59] WebAlert & WebSearch: fix kwalitee reported issues * Fixes various warnings and errors reported by kwalitee. --- modules/webalert/lib/webalert.py | 10 +-- modules/webalert/lib/webalert_templates.py | 85 +++++++++---------- modules/webalert/lib/webalert_webinterface.py | 2 +- modules/websearch/lib/websearch_templates.py | 79 +++++++++-------- .../websearch/lib/websearch_webinterface.py | 4 +- .../websearch/lib/websearch_yoursearches.py | 5 +- 6 files changed, 83 insertions(+), 102 deletions(-) diff --git a/modules/webalert/lib/webalert.py b/modules/webalert/lib/webalert.py index 936c7c355d..fe94b7db47 100644 --- a/modules/webalert/lib/webalert.py +++ b/modules/webalert/lib/webalert.py @@ -31,7 +31,7 @@ from invenio.webbasket import create_personal_baskets_selection_box from invenio.webbasket_dblayer import check_user_owns_baskets from invenio.messages import gettext_set_language -from invenio.dateutils import convert_datestruct_to_datetext, convert_datetext_to_dategui +from invenio.dateutils import convert_datestruct_to_datetext import invenio.template webalert_templates = invenio.template.load('webalert') @@ -219,7 +219,6 @@ def perform_request_youralerts_display(uid, search_clause_urlargs = [] search_clause_alert_name = [] if p: - p_stripped = p.strip() p_stripped_args = p.split() sql_p_stripped_args = ['\'%%' + quote(p_stripped_arg).replace('%','%%') + '%%\'' for p_stripped_arg in p_stripped_args] for sql_p_stripped_arg in sql_p_stripped_args: @@ -228,9 +227,6 @@ def perform_request_youralerts_display(uid, search_clause = "((%s) OR (%s))" % (" AND ".join(search_clause_urlargs), " AND ".join(search_clause_alert_name)) - idq_and_search_clause_list = [clause for clause in (idq_clause, search_clause) if clause] - idq_and_search_clause = ' AND '.join(idq_and_search_clause_list) - query_nb_alerts = """ SELECT COUNT(IF((uqb.id_user=%%s %s),uqb.id_query,NULL)), COUNT(q.id) @@ -343,9 +339,7 @@ def perform_request_youralerts_display(uid, page=page, step=step, paging_navigation=paging_navigation, - p=p, - guest=isGuestUser(uid), - guesttxt=warning_guest_user(type="alerts", ln=ln)) + p=p) return out def perform_remove_alert(alert_name, id_query, id_basket, uid, ln=CFG_SITE_LANG): diff --git a/modules/webalert/lib/webalert_templates.py b/modules/webalert/lib/webalert_templates.py index 2322b340b2..6b227588cd 100644 --- a/modules/webalert/lib/webalert_templates.py +++ b/modules/webalert/lib/webalert_templates.py @@ -270,9 +270,7 @@ def tmpl_youralerts_display(self, page, step, paging_navigation, - p, - guest, - guesttxt): + p): """ Displays an HTML formatted list of the user alerts. If the user has specified a query id, only the user alerts based on that @@ -297,13 +295,6 @@ def tmpl_youralerts_display(self, @param idq: The specified query id for which to display the user alerts @type idq: int - - @param guest: Whether the user is a guest or not - @type guest: boolean - - @param guesttxt: The HTML content of the warning box for guest users - (produced by webaccount.tmpl_warning_guest_user) - @type guesttxt: string """ # load the right message language @@ -794,51 +785,51 @@ def get_html_user_friendly_alert_query_args(args, _ = gettext_set_language(ln) # Arguments dictionary - dict = parse_qs(args) + args_dict = parse_qs(args) - if not dict.has_key('p') and not dict.has_key('p1') and not dict.has_key('p2') and not dict.has_key('p3'): + if not args_dict.has_key('p') and not args_dict.has_key('p1') and not args_dict.has_key('p2') and not args_dict.has_key('p3'): search_patterns_html = _('Searching for everything') else: search_patterns_html = _('Searching for') + ' ' - if dict.has_key('p'): - search_patterns_html += '' + cgi.escape(dict['p'][0]) + '' - if dict.has_key('f'): - search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(dict['f'][0]) + '' - if dict.has_key('p1'): - if dict.has_key('p'): + if args_dict.has_key('p'): + search_patterns_html += '' + cgi.escape(args_dict['p'][0]) + '' + if args_dict.has_key('f'): + search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(args_dict['f'][0]) + '' + if args_dict.has_key('p1'): + if args_dict.has_key('p'): search_patterns_html += ' ' + _('and') + ' ' - search_patterns_html += '' + cgi.escape(dict['p1'][0]) + '' - if dict.has_key('f1'): - search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(dict['f1'][0]) + '' - if dict.has_key('p2'): - if dict.has_key('p') or dict.has_key('p1'): - if dict.has_key('op1'): - search_patterns_html += ' %s ' % (dict['op1'][0] == 'a' and _('and') or \ - dict['op1'][0] == 'o' and _('or') or \ - dict['op1'][0] == 'n' and _('and not') or + search_patterns_html += '' + cgi.escape(args_dict['p1'][0]) + '' + if args_dict.has_key('f1'): + search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(args_dict['f1'][0]) + '' + if args_dict.has_key('p2'): + if args_dict.has_key('p') or args_dict.has_key('p1'): + if args_dict.has_key('op1'): + search_patterns_html += ' %s ' % (args_dict['op1'][0] == 'a' and _('and') or \ + args_dict['op1'][0] == 'o' and _('or') or \ + args_dict['op1'][0] == 'n' and _('and not') or ', ',) - search_patterns_html += '' + cgi.escape(dict['p2'][0]) + '' - if dict.has_key('f2'): - search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(dict['f2'][0]) + '' - if dict.has_key('p3'): - if dict.has_key('p') or dict.has_key('p1') or dict.has_key('p2'): - if dict.has_key('op2'): - search_patterns_html += ' %s ' % (dict['op2'][0] == 'a' and _('and') or \ - dict['op2'][0] == 'o' and _('or') or \ - dict['op2'][0] == 'n' and _('and not') or + search_patterns_html += '' + cgi.escape(args_dict['p2'][0]) + '' + if args_dict.has_key('f2'): + search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(args_dict['f2'][0]) + '' + if args_dict.has_key('p3'): + if args_dict.has_key('p') or args_dict.has_key('p1') or args_dict.has_key('p2'): + if args_dict.has_key('op2'): + search_patterns_html += ' %s ' % (args_dict['op2'][0] == 'a' and _('and') or \ + args_dict['op2'][0] == 'o' and _('or') or \ + args_dict['op2'][0] == 'n' and _('and not') or ', ',) - search_patterns_html += '' + cgi.escape(dict['p3'][0]) + '' - if dict.has_key('f3'): - search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(dict['f3'][0]) + '' + search_patterns_html += '' + cgi.escape(args_dict['p3'][0]) + '' + if args_dict.has_key('f3'): + search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(args_dict['f3'][0]) + '' - if not dict.has_key('c') and not dict.has_key('cc'): + if not args_dict.has_key('c') and not args_dict.has_key('cc'): collections_html = _('in all the collections') else: collections_html = _('in the following collection(s)') + ': ' - if dict.has_key('c'): - collections_html += ', '.join('' + cgi.escape(collection) + '' for collection in dict['c']) - elif dict.has_key('cc'): - collections_html += '' + cgi.escape(dict['cc'][0]) + '' + if args_dict.has_key('c'): + collections_html += ', '.join('' + cgi.escape(collection) + '' for collection in args_dict['c']) + elif args_dict.has_key('cc'): + collections_html += '' + cgi.escape(args_dict['cc'][0]) + '' search_query_args_html = search_patterns_html + '
    ' + collections_html @@ -884,9 +875,9 @@ def get_html_user_friendly_date_from_datetext(given_date, if given_date_datestruct[0] != 0 and \ given_date_datestruct[1] != 0 and \ given_date_datestruct[2] != 0: - days_old = (today - datetime_date(given_date_datestruct.tm_year, - given_date_datestruct.tm_mon, - given_date_datestruct.tm_mday)).days + days_old = (today - datetime_date(given_date_datestruct[0], + given_date_datestruct[1], + given_date_datestruct[2])).days if days_old == 0: out = _('Today') elif days_old < 7: diff --git a/modules/webalert/lib/webalert_webinterface.py b/modules/webalert/lib/webalert_webinterface.py index 6c2c850d99..17c23b014d 100644 --- a/modules/webalert/lib/webalert_webinterface.py +++ b/modules/webalert/lib/webalert_webinterface.py @@ -60,7 +60,7 @@ def index(self, req, dummy): redirect_to_url(req, '%s/youralerts/display' % CFG_SITE_SECURE_URL) - def list(self, req, form): + def list(self, req, dummy): """ Legacy youralerts list page. Now redirects to the youralerts display page. diff --git a/modules/websearch/lib/websearch_templates.py b/modules/websearch/lib/websearch_templates.py index 46e65902a7..8a9bfa715b 100644 --- a/modules/websearch/lib/websearch_templates.py +++ b/modules/websearch/lib/websearch_templates.py @@ -5092,8 +5092,6 @@ def tmpl_yoursearches_display(self, step, paging_navigation, p, - guest, - guesttxt, ln=CFG_SITE_LANG): """ Template for the display the user's search history. @@ -5256,8 +5254,7 @@ def tmpl_yoursearches_display(self, %(yoursearches)s -""" % {'header': _('Search details'), - 'footer': footer, +""" % {'footer': footer, 'yoursearches': yoursearches} return out @@ -5283,51 +5280,51 @@ def get_html_user_friendly_search_query_args(args, _ = gettext_set_language(ln) # Arguments dictionary - dict = parse_qs(args) + args_dict = parse_qs(args) - if not dict.has_key('p') and not dict.has_key('p1') and not dict.has_key('p2') and not dict.has_key('p3'): + if not args_dict.has_key('p') and not args_dict.has_key('p1') and not args_dict.has_key('p2') and not args_dict.has_key('p3'): search_patterns_html = _('Search for everything') else: search_patterns_html = _('Search for') + ' ' - if dict.has_key('p'): - search_patterns_html += '' + cgi.escape(dict['p'][0]) + '' - if dict.has_key('f'): - search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(dict['f'][0]) + '' - if dict.has_key('p1'): - if dict.has_key('p'): + if args_dict.has_key('p'): + search_patterns_html += '' + cgi.escape(args_dict['p'][0]) + '' + if args_dict.has_key('f'): + search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(args_dict['f'][0]) + '' + if args_dict.has_key('p1'): + if args_dict.has_key('p'): search_patterns_html += ' ' + _('and') + ' ' - search_patterns_html += '' + cgi.escape(dict['p1'][0]) + '' - if dict.has_key('f1'): - search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(dict['f1'][0]) + '' - if dict.has_key('p2'): - if dict.has_key('p') or dict.has_key('p1'): - if dict.has_key('op1'): - search_patterns_html += ' %s ' % (dict['op1'][0] == 'a' and _('and') or \ - dict['op1'][0] == 'o' and _('or') or \ - dict['op1'][0] == 'n' and _('and not') or + search_patterns_html += '' + cgi.escape(args_dict['p1'][0]) + '' + if args_dict.has_key('f1'): + search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(args_dict['f1'][0]) + '' + if args_dict.has_key('p2'): + if args_dict.has_key('p') or args_dict.has_key('p1'): + if args_dict.has_key('op1'): + search_patterns_html += ' %s ' % (args_dict['op1'][0] == 'a' and _('and') or \ + args_dict['op1'][0] == 'o' and _('or') or \ + args_dict['op1'][0] == 'n' and _('and not') or ', ',) - search_patterns_html += '' + cgi.escape(dict['p2'][0]) + '' - if dict.has_key('f2'): - search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(dict['f2'][0]) + '' - if dict.has_key('p3'): - if dict.has_key('p') or dict.has_key('p1') or dict.has_key('p2'): - if dict.has_key('op2'): - search_patterns_html += ' %s ' % (dict['op2'][0] == 'a' and _('and') or \ - dict['op2'][0] == 'o' and _('or') or \ - dict['op2'][0] == 'n' and _('and not') or + search_patterns_html += '' + cgi.escape(args_dict['p2'][0]) + '' + if args_dict.has_key('f2'): + search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(args_dict['f2'][0]) + '' + if args_dict.has_key('p3'): + if args_dict.has_key('p') or args_dict.has_key('p1') or args_dict.has_key('p2'): + if args_dict.has_key('op2'): + search_patterns_html += ' %s ' % (args_dict['op2'][0] == 'a' and _('and') or \ + args_dict['op2'][0] == 'o' and _('or') or \ + args_dict['op2'][0] == 'n' and _('and not') or ', ',) - search_patterns_html += '' + cgi.escape(dict['p3'][0]) + '' - if dict.has_key('f3'): - search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(dict['f3'][0]) + '' + search_patterns_html += '' + cgi.escape(args_dict['p3'][0]) + '' + if args_dict.has_key('f3'): + search_patterns_html += ' ' + _('as') + ' ' + '' + cgi.escape(args_dict['f3'][0]) + '' - if not dict.has_key('c') and not dict.has_key('cc'): + if not args_dict.has_key('c') and not args_dict.has_key('cc'): collections_html = _('in all the collections') else: collections_html = _('in the following collection(s)') + ': ' - if dict.has_key('c'): - collections_html += ', '.join('' + cgi.escape(collection) + '' for collection in dict['c']) - elif dict.has_key('cc'): - collections_html += '' + cgi.escape(dict['cc'][0]) + '' + if args_dict.has_key('c'): + collections_html += ', '.join('' + cgi.escape(collection) + '' for collection in args_dict['c']) + elif args_dict.has_key('cc'): + collections_html += '' + cgi.escape(args_dict['cc'][0]) + '' search_query_args_html = search_patterns_html + '
    ' + collections_html @@ -5373,9 +5370,9 @@ def get_html_user_friendly_date_from_datetext(given_date, if given_date_datestruct[0] != 0 and \ given_date_datestruct[1] != 0 and \ given_date_datestruct[2] != 0: - days_old = (today - datetime_date(given_date_datestruct.tm_year, - given_date_datestruct.tm_mon, - given_date_datestruct.tm_mday)).days + days_old = (today - datetime_date(given_date_datestruct[0], + given_date_datestruct[1], + given_date_datestruct[2])).days if days_old == 0: out = _('Today') elif days_old < 7: diff --git a/modules/websearch/lib/websearch_webinterface.py b/modules/websearch/lib/websearch_webinterface.py index 9f779e46bc..6be0da4fcc 100644 --- a/modules/websearch/lib/websearch_webinterface.py +++ b/modules/websearch/lib/websearch_webinterface.py @@ -113,6 +113,7 @@ from invenio.bibfield import get_record from invenio.shellutils import mymkdir from invenio.websearch_yoursearches import perform_request_yoursearches_display +from invenio.webstat import register_customevent import invenio.template websearch_templates = invenio.template.load('websearch') @@ -1209,8 +1210,9 @@ class WebInterfaceYourSearchesPages(WebInterfaceDirectory): _exports = ['', 'display'] - def index(self, req, form): + def index(self, req, dummy): """ + Redirects the user to the display page. """ redirect_to_url(req, '%s/yoursearches/display' % CFG_SITE_SECURE_URL) diff --git a/modules/websearch/lib/websearch_yoursearches.py b/modules/websearch/lib/websearch_yoursearches.py index 23e9c549d4..ddb74a6566 100644 --- a/modules/websearch/lib/websearch_yoursearches.py +++ b/modules/websearch/lib/websearch_yoursearches.py @@ -24,7 +24,7 @@ from invenio.webaccount import warning_guest_user from invenio.messages import gettext_set_language from invenio.webuser import isGuestUser -from urllib import quote, quote_plus, unquote_plus +from urllib import quote from invenio.webalert import count_user_alerts_for_given_query import invenio.template @@ -62,7 +62,6 @@ def perform_request_yoursearches_display(uid, search_clause = "" if p: - p_stripped = p.strip() p_stripped_args = p.split() sql_p_stripped_args = ['\'%%' + quote(p_stripped_arg).replace('%','%%') + '%%\'' for p_stripped_arg in p_stripped_args] for sql_p_stripped_arg in sql_p_stripped_args: @@ -149,8 +148,6 @@ def perform_request_yoursearches_display(uid, step=step, paging_navigation=paging_navigation, p=p, - guest = isGuestUser(uid), - guesttxt = warning_guest_user(type="searches", ln=ln), ln = ln) def account_list_searches(uid, From 8f8ada917f47c6c6c0a259e27498622b9c32f383 Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Wed, 24 Jul 2013 12:59:55 +0200 Subject: [PATCH 26/59] WebStyle: add new images to the Makefile * Adds new images for yoursearches and youralerts to the Makefile --- modules/webstyle/img/Makefile.am | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/webstyle/img/Makefile.am b/modules/webstyle/img/Makefile.am index 57bb29d3d1..169dcb0655 100644 --- a/modules/webstyle/img/Makefile.am +++ b/modules/webstyle/img/Makefile.am @@ -326,7 +326,13 @@ img_DATA = add-small.png \ yahoo_icon_24.png \ yahoo_icon_48.png \ yammer_icon_24.png \ - yammer_icon_48.png + yammer_icon_48.png \ + yoursearches_alert.png \ + yoursearches_alert_edit.png \ + yoursearches_search.png \ + youralerts_alert.png \ + youralerts_alert_delete.png \ + youralerts_alert_edit.png tmpdir=$(localstatedir)/tmp From 08ccec4ebcec02dcd1430e15e488486a4762c63c Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Fri, 8 Nov 2013 09:12:15 +0100 Subject: [PATCH 27/59] WebAlert: option to pause and resume alerts * Adds the option to pause and resume alerts. The option can be configured either in when editing a specific alert or when displaying all the alerts. (closes #1227) --- ...13_12_11_new_is_active_colum_for_alerts.py | 34 +++ modules/miscutil/sql/tabcreate.sql | 1 + modules/webalert/lib/alert_engine.py | 26 ++- modules/webalert/lib/webalert.py | 154 +++++++++++-- modules/webalert/lib/webalert_templates.py | 113 +++++++--- modules/webalert/lib/webalert_webinterface.py | 208 +++++++++++++++++- modules/webstyle/css/invenio.css | 28 ++- modules/webstyle/img/Makefile.am | 4 +- .../webstyle/img/youralerts_alert_pause.png | Bin 0 -> 1085 bytes .../webstyle/img/youralerts_alert_resume.png | Bin 0 -> 1011 bytes 10 files changed, 488 insertions(+), 80 deletions(-) create mode 100644 modules/miscutil/lib/upgrades/invenio_2013_12_11_new_is_active_colum_for_alerts.py create mode 100644 modules/webstyle/img/youralerts_alert_pause.png create mode 100644 modules/webstyle/img/youralerts_alert_resume.png diff --git a/modules/miscutil/lib/upgrades/invenio_2013_12_11_new_is_active_colum_for_alerts.py b/modules/miscutil/lib/upgrades/invenio_2013_12_11_new_is_active_colum_for_alerts.py new file mode 100644 index 0000000000..163daf8b79 --- /dev/null +++ b/modules/miscutil/lib/upgrades/invenio_2013_12_11_new_is_active_colum_for_alerts.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +## +## This file is part of Invenio. +## Copyright (C) 2013 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +from invenio.dbquery import run_sql + +depends_on = ['invenio_release_1_1_0'] + +def info(): + return "New is_active column for the user_query_basket table" + +def do_upgrade(): + run_sql(""" +ALTER TABLE user_query_basket ADD COLUMN is_active BOOL DEFAULT 1 AFTER notification; +""") + +def estimate(): + """ Estimate running time of upgrade in seconds (optional). """ + return 1 diff --git a/modules/miscutil/sql/tabcreate.sql b/modules/miscutil/sql/tabcreate.sql index 648f3fe4e2..583ab66838 100644 --- a/modules/miscutil/sql/tabcreate.sql +++ b/modules/miscutil/sql/tabcreate.sql @@ -3749,6 +3749,7 @@ CREATE TABLE IF NOT EXISTS user_query_basket ( alert_desc text default NULL, alert_recipient text default NULL, notification char(1) NOT NULL default 'y', + is_active tinyint(1) DEFAULT '1', PRIMARY KEY (id_user,id_query,frequency,id_basket), KEY alert_name (alert_name) ) ENGINE=MyISAM; diff --git a/modules/webalert/lib/alert_engine.py b/modules/webalert/lib/alert_engine.py index fb35bf5944..1e8f91b34b 100644 --- a/modules/webalert/lib/alert_engine.py +++ b/modules/webalert/lib/alert_engine.py @@ -63,15 +63,23 @@ def update_date_lastrun(alert): return run_sql('update user_query_basket set date_lastrun=%s where id_user=%s and id_query=%s and id_basket=%s;', (strftime("%Y-%m-%d"), alert[0], alert[1], alert[2],)) -def get_alert_queries(frequency): - """Return all the queries for the given frequency.""" - - return run_sql('select distinct id, urlargs from query q, user_query_basket uqb where q.id=uqb.id_query and uqb.frequency=%s and uqb.date_lastrun <= now();', (frequency,)) - -def get_alert_queries_for_user(uid): - """Returns all the queries for the given user id.""" - - return run_sql('select distinct id, urlargs, uqb.frequency from query q, user_query_basket uqb where q.id=uqb.id_query and uqb.id_user=%s and uqb.date_lastrun <= now();', (uid,)) +def get_alert_queries(frequency, only_active=True): + """ + Return all the active alert queries for the given frequency. + If only_active is False: fetch all the alert queries + regardless of them being active or not. + """ + + return run_sql('select distinct id, urlargs from query q, user_query_basket uqb where q.id=uqb.id_query and uqb.frequency=%%s and uqb.date_lastrun <= now()%s;' % (only_active and ' and is_active = 1' or '', ), (frequency, )) + +def get_alert_queries_for_user(uid, only_active=True): + """ + Returns all the active alert queries for the given user id. + If only_active is False: fetch all the alert queries + regardless of them being active or not. + """ + + return run_sql('select distinct id, urlargs, uqb.frequency from query q, user_query_basket uqb where q.id=uqb.id_query and uqb.id_user=%%s and uqb.date_lastrun <= now()%s;' % (only_active and ' and is_active = 1' or '', ), (uid, )) def get_alerts(query, frequency): """Returns a dictionary of all the records found for a specific query and frequency along with other informationm""" diff --git a/modules/webalert/lib/webalert.py b/modules/webalert/lib/webalert.py index fe94b7db47..2037e1cff6 100644 --- a/modules/webalert/lib/webalert.py +++ b/modules/webalert/lib/webalert.py @@ -88,7 +88,16 @@ def check_user_can_add_alert(id_user, id_query): return True return False -def perform_input_alert(action, id_query, alert_name, frequency, notification, id_basket, uid, old_id_basket=None, ln = CFG_SITE_LANG): +def perform_input_alert(action, + id_query, + alert_name, + frequency, + notification, + id_basket, + uid, + is_active, + old_id_basket=None, + ln = CFG_SITE_LANG): """get the alert settings input: action="add" for a new alert (blank form), action="modify" for an update (get old values) @@ -96,21 +105,29 @@ def perform_input_alert(action, id_query, alert_name, frequency, notification, i for the "modify" action specify old alert_name, frequency of checking, e-mail notification and basket id. output: alert settings input form""" + # load the right language _ = gettext_set_language(ln) + # security check: if not check_user_can_add_alert(uid, id_query): raise AlertError(_("You do not have rights for this operation.")) + + # normalize is_active (it should be either 1 (True) or 0 (False)) + is_active = is_active and 1 or 0 + # display query information res = run_sql("SELECT urlargs FROM query WHERE id=%s", (id_query,)) try: urlargs = res[0][0] except: urlargs = "UNKNOWN" + baskets = create_personal_baskets_selection_box(uid=uid, html_select_box_name='idb', selected_bskid=old_id_basket, ln=ln) + return webalert_templates.tmpl_input_alert( ln = ln, query = get_textual_query_info_from_urlargs(urlargs, ln = ln), @@ -122,6 +139,7 @@ def perform_input_alert(action, id_query, alert_name, frequency, notification, i old_id_basket = old_id_basket, id_basket = id_basket, id_query = id_query, + is_active = is_active, guest = isGuestUser(uid), guesttxt = warning_guest_user(type="alerts", ln=ln) ) @@ -279,7 +297,8 @@ def perform_request_youralerts_display(uid, uqb.frequency, uqb.notification, DATE_FORMAT(uqb.date_creation,'%s'), - DATE_FORMAT(uqb.date_lastrun,'%s') + DATE_FORMAT(uqb.date_lastrun,'%s'), + uqb.is_active FROM user_query_basket uqb LEFT JOIN query q ON uqb.id_query=q.id @@ -305,7 +324,8 @@ def perform_request_youralerts_display(uid, alert_frequency, alert_notification, alert_creation, - alert_last_run) in result: + alert_last_run, + alert_is_active) in result: try: if not query_id: raise StandardError("""\ @@ -314,17 +334,18 @@ def perform_request_youralerts_display(uid, Please check this and delete it if needed. Otherwise no problem, I'm continuing with the other alerts now. Here are all the alerts defined by this user: %s""" % (uid, repr(result))) - alerts.append({'queryid' : query_id, - 'queryargs' : query_args, - 'textargs' : get_textual_query_info_from_urlargs(query_args, ln=ln), - 'userid' : uid, - 'basketid' : bsk_id, - 'basketname' : bsk_name, - 'alertname' : alert_name, - 'frequency' : alert_frequency, + alerts.append({'queryid' : query_id, + 'queryargs' : query_args, + 'textargs' : get_textual_query_info_from_urlargs(query_args, ln=ln), + 'userid' : uid, + 'basketid' : bsk_id, + 'basketname' : bsk_name, + 'alertname' : alert_name, + 'frequency' : alert_frequency, 'notification' : alert_notification, - 'created' : alert_creation, - 'lastrun' : alert_last_run}) + 'created' : alert_creation, + 'lastrun' : alert_last_run, + 'is_active' : alert_is_active}) except StandardError: register_exception(alert_admin=True) else: @@ -370,8 +391,91 @@ def perform_remove_alert(alert_name, id_query, id_basket, uid, ln=CFG_SITE_LANG) out += perform_request_youralerts_display(uid, idq=0, ln=ln) return out +def perform_pause_alert(alert_name, id_query, id_basket, uid, ln=CFG_SITE_LANG): + """Pause an alert + input: alert name + identifier of the query; + identifier of the basket + uid + output: confirmation message + the list of alerts Web page""" + + # load the right language + _ = gettext_set_language(ln) + + # security check: + if not check_user_can_add_alert(uid, id_query): + raise AlertError(_("You do not have rights for this operation.")) + + # set variables + out = "" + if (None in (alert_name, id_query, id_basket, uid)): + return out + + # DB call to pause the alert + query = """ UPDATE user_query_basket + SET is_active = 0 + WHERE id_user=%s + AND id_query=%s + AND id_basket=%s""" + params = (uid, id_query, id_basket) + res = run_sql(query, params) + + if res: + out += '

    %s

    ' % _('Alert successfully paused.') + else: + out += '

    %s

    ' % _('Unable to pause alert.') + + out += perform_request_youralerts_display(uid, idq=0, ln=ln) + + return out + +def perform_resume_alert(alert_name, id_query, id_basket, uid, ln=CFG_SITE_LANG): + """Resume an alert + input: alert name + identifier of the query; + identifier of the basket + uid + output: confirmation message + the list of alerts Web page""" -def perform_update_alert(alert_name, frequency, notification, id_basket, id_query, old_id_basket, uid, ln = CFG_SITE_LANG): + # load the right language + _ = gettext_set_language(ln) + + # security check: + if not check_user_can_add_alert(uid, id_query): + raise AlertError(_("You do not have rights for this operation.")) + + # set variables + out = "" + if (None in (alert_name, id_query, id_basket, uid)): + return out + + # DB call to resume the alert + query = """ UPDATE user_query_basket + SET is_active = 1 + WHERE id_user=%s + AND id_query=%s + AND id_basket=%s""" + params = (uid, id_query, id_basket) + res = run_sql(query, params) + + if res: + out += '

    %s

    ' % _('Alert successfully resumed.') + else: + out += '

    %s

    ' % _('Unable to resume alert.') + + out += perform_request_youralerts_display(uid, idq=0, ln=ln) + + return out + +def perform_update_alert(alert_name, + frequency, + notification, + id_basket, + id_query, + old_id_basket, + uid, + is_active, + ln = CFG_SITE_LANG): """update alert settings into the database input: the name of the new alert; alert frequency: 'month', 'week' or 'day'; @@ -383,9 +487,12 @@ def perform_update_alert(alert_name, frequency, notification, id_basket, id_quer output: confirmation message + the list of alerts Web page""" out = '' # sanity check - if (None in (alert_name, frequency, notification, id_basket, id_query, old_id_basket, uid)): + if (None in (alert_name, frequency, notification, id_basket, id_query, old_id_basket, uid, is_active)): return out + # normalize is_active (it should be either 1 (True) or 0 (False)) + is_active = is_active and 1 or 0 + # load the right language _ = gettext_set_language(ln) @@ -416,13 +523,20 @@ def perform_update_alert(alert_name, frequency, notification, id_basket, id_quer check_alert_is_unique( id_basket, id_query, uid, ln) # update a row into the alerts table: user_query_basket - query = """UPDATE user_query_basket - SET alert_name=%s,frequency=%s,notification=%s, - date_creation=%s,date_lastrun='',id_basket=%s - WHERE id_user=%s AND id_query=%s AND id_basket=%s""" + query = """ UPDATE user_query_basket + SET alert_name=%s, + frequency=%s, + notification=%s, + date_creation=%s, + date_lastrun='', + id_basket=%s, + is_active=%s + WHERE id_user=%s + AND id_query=%s + AND id_basket=%s""" params = (alert_name, frequency, notification, convert_datestruct_to_datetext(time.localtime()), - id_basket, uid, id_query, old_id_basket) + id_basket, is_active, uid, id_query, old_id_basket) run_sql(query, params) diff --git a/modules/webalert/lib/webalert_templates.py b/modules/webalert/lib/webalert_templates.py index 6b227588cd..ff398282bc 100644 --- a/modules/webalert/lib/webalert_templates.py +++ b/modules/webalert/lib/webalert_templates.py @@ -128,9 +128,20 @@ def tmpl_account_list_alerts(self, ln, alerts): } return out - def tmpl_input_alert(self, ln, query, alert_name, action, frequency, notification, - baskets, old_id_basket, id_basket, id_query, - guest, guesttxt): + def tmpl_input_alert(self, + ln, + query, + alert_name, + action, + frequency, + notification, + baskets, + old_id_basket, + id_basket, + id_query, + is_active, + guest, + guesttxt): """ Displays an alert adding form. @@ -156,6 +167,8 @@ def tmpl_input_alert(self, ln, query, alert_name, action, frequency, notificatio - 'id_query' *string* - The id of the query associated to this alert + - 'is_active' *boolean* - is the alert active or not + - 'guest' *bool* - If the user is a guest user - 'guesttxt' *string* - The HTML content of the warning box for guest users (produced by webaccount.tmpl_warning_guest_user) @@ -197,6 +210,10 @@ def tmpl_input_alert(self, ln, query, alert_name, action, frequency, notificatio + + %(is_active_label)s + + %(send_email)s @@ -209,7 +226,8 @@ def tmpl_input_alert(self, ln, query, alert_name, action, frequency, notificatio %(store_basket)s - %(baskets)s + %(baskets)s + """ % { 'action': action, 'alert_name' : _("Alert identification name:"), @@ -229,29 +247,27 @@ def tmpl_input_alert(self, ln, query, alert_name, action, frequency, notificatio 'specify' : _("if %(x_fmt_open)sno%(x_fmt_close)s you must specify a basket") % {'x_fmt_open': '', 'x_fmt_close': ''}, 'store_basket' : _("Store results in basket?"), - 'baskets': baskets + 'baskets': baskets, + 'is_active_label' : _("Is the alert active?"), + 'is_active_checkbox' : is_active and 'checked="checked" ' or '', } - out += """ - - + out += """   - - - - - + + - """ % { - 'idq' : id_query, - 'ln' : ln, - 'set_alert' : _("SET ALERT"), - 'clear_data' : _("CLEAR DATA"), - } + + + """ % {'idq' : id_query, + 'ln' : ln, + 'set_alert' : _("SET ALERT"), + 'clear_data' : _("CLEAR DATA"),} + if action == "update": out += '' % old_id_basket out += "" @@ -291,6 +307,7 @@ def tmpl_youralerts_display(self, 'notification' *string* - If notification should be sent by email ('y', 'n') 'created' *string* - The date of alert creation 'lastrun' *string* - The last running date + 'is_active' *boolean* - is the alert active or not @type alerts: list of dictionaries @param idq: The specified query id for which to display the user alerts @@ -379,7 +396,7 @@ def tmpl_youralerts_display(self, msg = _('You have defined a total of %(number_of_alerts)s alerts.') % \ {'number_of_alerts': '' + str(nb_alerts) + ''} msg += '
    ' - msg += _('You may define new alert based on %(yoursearches)s, the %(popular_alerts)s or just by %(search_interface)s.') % \ + msg += _('You may define new alerts based on %(yoursearches)s, the %(popular_alerts)s or just by %(search_interface)s.') % \ {'yoursearches': '%s' % (CFG_SITE_SECURE_URL, ln, _('your searches')), 'popular_alerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('popular alerts')), 'search_interface': '%s' %(CFG_SITE_URL, ln, _('searching for something new'))} @@ -417,6 +434,7 @@ def tmpl_youralerts_display(self, alert_notification = alert['notification'] alert_creation_date = alert['created'] alert_last_run_date = alert['lastrun'] + alert_active_p = alert['is_active'] alert_details_frequency = _('Runs') + ' ' + \ (alert_frequency == 'day' and '' + _('daily') + '' or \ @@ -444,16 +462,26 @@ def tmpl_youralerts_display(self, _('Last run:') + ' ' + \ alert_details_last_run_date + alert_details_options_pause_or_resume = create_html_link('%s/youralerts/%s' % \ + (CFG_SITE_SECURE_URL, alert_active_p and 'pause' or 'resume'), + {'ln' : ln, + 'idq' : alert_query_id, + 'name' : alert_name, + 'idb' : alert_basket_id,}, + alert_active_p and _('Pause') or _('Resume')) + alert_details_options_edit = create_html_link('%s/youralerts/modify' % \ (CFG_SITE_SECURE_URL,), - {'ln' : ln, - 'idq' : alert_query_id, - 'name' : alert_name, - 'freq' : alert_frequency, - 'notif' : alert_notification, - 'idb' : alert_basket_id, - 'old_idb': alert_basket_id}, + {'ln' : ln, + 'idq' : alert_query_id, + 'name' : alert_name, + 'freq' : alert_frequency, + 'notif' : alert_notification, + 'idb' : alert_basket_id, + 'is_active' : alert_active_p, + 'old_idb' : alert_basket_id}, _('Edit')) + alert_details_options_delete = create_html_link('%s/youralerts/remove' % \ (CFG_SITE_SECURE_URL,), {'ln' : ln, @@ -463,10 +491,18 @@ def tmpl_youralerts_display(self, _('Delete'), {'onclick': 'return confirm(\'%s\')' % \ (_('Are you sure you want to permanently delete this alert?'),)}) - alert_details_options = '' % (CFG_SITE_URL,) + \ + + # TODO: find a nice way to format the display alert options + alert_details_options = '' % \ + (CFG_SITE_URL, alert_active_p and 'pause' or 'resume') + \ + alert_details_options_pause_or_resume + \ + ' · ' + \ + ' ' % \ + (CFG_SITE_URL,) + \ alert_details_options_edit + \ - '   ' + \ - '' % (CFG_SITE_URL,) + \ + ' · ' + \ + '' % \ + (CFG_SITE_URL,) + \ alert_details_options_delete youralerts_display_html += """ @@ -475,23 +511,28 @@ def tmpl_youralerts_display(self, %(counter)i. -
    -
    %(alert_name)s
    +
    +
    %(warning_label_is_active_p)s%(alert_name)s
    %(alert_details_frequency_notification_basket)s
    %(alert_details_search_query)s
    -
    +
    +
    %(alert_details_options)s
    -
    -
    %(alert_details_creation_last_run_dates)s
    +
    +
    %(alert_details_creation_last_run_dates)s
    +
    """ % {'counter': counter, 'alert_name': cgi.escape(alert_name), 'alert_details_frequency_notification_basket': alert_details_frequency_notification_basket, 'alert_details_search_query': alert_details_search_query, 'alert_details_options': alert_details_options, - 'alert_details_creation_last_run_dates': alert_details_creation_last_run_dates} + 'alert_details_creation_last_run_dates': alert_details_creation_last_run_dates, + 'css_class_content_is_active_p' : not alert_active_p and ' youralerts_display_table_content_inactive' or '', + 'warning_label_is_active_p' : not alert_active_p and '[ %s ] ' % _('paused') or '', + } footer = '' if paging_navigation[0]: diff --git a/modules/webalert/lib/webalert_webinterface.py b/modules/webalert/lib/webalert_webinterface.py index 17c23b014d..20edb2279f 100644 --- a/modules/webalert/lib/webalert_webinterface.py +++ b/modules/webalert/lib/webalert_webinterface.py @@ -22,13 +22,15 @@ __lastupdated__ = """$Date$""" from invenio.config import CFG_SITE_SECURE_URL, CFG_SITE_NAME, \ - CFG_ACCESS_CONTROL_LEVEL_SITE, CFG_SITE_NAME_INTL + CFG_ACCESS_CONTROL_LEVEL_SITE, CFG_SITE_NAME_INTL, CFG_SITE_LANG from invenio.webpage import page from invenio.webalert import perform_input_alert, \ perform_request_youralerts_display, \ perform_add_alert, \ perform_update_alert, \ perform_remove_alert, \ + perform_pause_alert, \ + perform_resume_alert, \ perform_request_youralerts_popular, \ AlertError from invenio.webuser import getUid, page_not_authorized, isGuestUser @@ -53,6 +55,8 @@ class WebInterfaceYourAlertsPages(WebInterfaceDirectory): 'add', 'update', 'remove', + 'pause', + 'resume', 'popular'] def index(self, req, dummy): @@ -101,8 +105,15 @@ def input(self, req, form): text = _("You are not authorized to use alerts.")) try: - html = perform_input_alert("add", argd['idq'], argd['name'], argd['freq'], - argd['notif'], argd['idb'], uid, ln=argd['ln']) + html = perform_input_alert("add", + argd['idq'], + argd['name'], + argd['freq'], + argd['notif'], + argd['idb'], + uid, + is_active = 1, + ln = argd['ln']) except AlertError, msg: return page(title=_("Error"), body=webalert_templates.tmpl_errorMsg(ln=argd['ln'], error_msg=msg), @@ -160,6 +171,7 @@ def modify(self, req, form): 'freq': (str, "week"), 'notif': (str, "y"), 'idb': (int, 0), + 'is_active': (int, 0), 'error_msg': (str, ""), }) @@ -186,8 +198,15 @@ def modify(self, req, form): text = _("You are not authorized to use alerts.")) try: - html = perform_input_alert("update", argd['idq'], argd['name'], argd['freq'], - argd['notif'], argd['idb'], uid, argd['old_idb'], ln=argd['ln']) + html = perform_input_alert("update", argd['idq'], + argd['name'], + argd['freq'], + argd['notif'], + argd['idb'], + uid, + argd['is_active'], + argd['old_idb'], + ln=argd['ln']) except AlertError, msg: return page(title=_("Error"), body=webalert_templates.tmpl_errorMsg(ln=argd['ln'], error_msg=msg), @@ -379,6 +398,7 @@ def update(self, req, form): 'notif': (str, None), 'idb': (int, None), 'idq': (int, None), + 'is_active': (int, 0), 'old_idb': (int, None), }) @@ -405,8 +425,15 @@ def update(self, req, form): text = _("You are not authorized to use alerts.")) try: - html = perform_update_alert(argd['name'], argd['freq'], argd['notif'], - argd['idb'], argd['idq'], argd['old_idb'], uid, ln=argd['ln']) + html = perform_update_alert(argd['name'], + argd['freq'], + argd['notif'], + argd['idb'], + argd['idq'], + argd['old_idb'], + uid, + argd['is_active'], + ln=argd['ln']) except AlertError, msg: return page(title=_("Error"), body=webalert_templates.tmpl_errorMsg(ln=argd['ln'], error_msg=msg), @@ -450,6 +477,9 @@ def update(self, req, form): navmenuid='youralerts') def remove(self, req, form): + """ + Remove an alert from the DB. + """ argd = wash_urlargd(form, {'name': (str, None), 'idq': (int, None), @@ -525,6 +555,170 @@ def remove(self, req, form): lastupdated=__lastupdated__, navmenuid='youralerts') + def pause(self, req, form): + """ + Pause an alert. + """ + + argd = wash_urlargd(form, {'name' : (str, None), + 'idq' : (int, None), + 'idb' : (int, None), + 'ln' : (str, CFG_SITE_LANG), + }) + + uid = getUid(req) + + if CFG_ACCESS_CONTROL_LEVEL_SITE >= 1: + return page_not_authorized(req, "%s/youralerts/pause" % \ + (CFG_SITE_SECURE_URL,), + navmenuid="youralerts") + elif uid == -1 or isGuestUser(uid): + return redirect_to_url(req, "%s/youraccount/login%s" % ( + CFG_SITE_SECURE_URL, + make_canonical_urlargd({ + 'referer' : "%s/youralerts/pause%s" % ( + CFG_SITE_SECURE_URL, + make_canonical_urlargd(argd, {})), + "ln" : argd['ln']}, {}))) + + # load the right language + _ = gettext_set_language(argd['ln']) + user_info = collect_user_info(req) + if not user_info['precached_usealerts']: + return page_not_authorized(req, "../", \ + text = _("You are not authorized to use alerts.")) + + try: + html = perform_pause_alert(argd['name'], + argd['idq'], + argd['idb'], + uid, + ln=argd['ln']) + except AlertError, msg: + return page(title=_("Error"), + body=webalert_templates.tmpl_errorMsg(ln=argd['ln'], error_msg=msg), + navtrail= """%(account)s""" % { + 'sitesecureurl' : CFG_SITE_SECURE_URL, + 'ln': argd['ln'], + 'account' : _("Your Account"), + }, + description=_("%s Personalize, Set a new alert") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME), + keywords=_("%s, personalize") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME), + uid=uid, + language=argd['ln'], + req=req, + lastupdated=__lastupdated__, + navmenuid='youralerts') + + # register event in webstat + alert_str = "%s (%d)" % (argd['name'], argd['idq']) + if user_info['email']: + user_str = "%s (%d)" % (user_info['email'], user_info['uid']) + else: + user_str = "" + try: + register_customevent("alerts", ["pause", alert_str, user_str]) + except: + register_exception(suffix="Do the webstat tables exists? Try with 'webstatadmin --load-config'") + + # display success + return page(title=_("Display alerts"), + body=html, + navtrail= """%(account)s""" % { + 'sitesecureurl' : CFG_SITE_SECURE_URL, + 'ln': argd['ln'], + 'account' : _("Your Account"), + }, + description=_("%s Personalize, Display alerts") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME), + keywords=_("%s, personalize") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME), + uid=uid, + language=argd['ln'], + req=req, + lastupdated=__lastupdated__, + navmenuid='youralerts') + + def resume(self, req, form): + """ + Resume an alert. + """ + + argd = wash_urlargd(form, {'name' : (str, None), + 'idq' : (int, None), + 'idb' : (int, None), + 'ln' : (str, CFG_SITE_LANG), + }) + + uid = getUid(req) + + if CFG_ACCESS_CONTROL_LEVEL_SITE >= 1: + return page_not_authorized(req, "%s/youralerts/resume" % \ + (CFG_SITE_SECURE_URL,), + navmenuid="youralerts") + elif uid == -1 or isGuestUser(uid): + return redirect_to_url(req, "%s/youraccount/login%s" % ( + CFG_SITE_SECURE_URL, + make_canonical_urlargd({ + 'referer' : "%s/youralerts/resume%s" % ( + CFG_SITE_SECURE_URL, + make_canonical_urlargd(argd, {})), + "ln" : argd['ln']}, {}))) + + # load the right language + _ = gettext_set_language(argd['ln']) + user_info = collect_user_info(req) + if not user_info['precached_usealerts']: + return page_not_authorized(req, "../", \ + text = _("You are not authorized to use alerts.")) + + try: + html = perform_resume_alert(argd['name'], + argd['idq'], + argd['idb'], + uid, + ln=argd['ln']) + except AlertError, msg: + return page(title=_("Error"), + body=webalert_templates.tmpl_errorMsg(ln=argd['ln'], error_msg=msg), + navtrail= """%(account)s""" % { + 'sitesecureurl' : CFG_SITE_SECURE_URL, + 'ln': argd['ln'], + 'account' : _("Your Account"), + }, + description=_("%s Personalize, Set a new alert") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME), + keywords=_("%s, personalize") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME), + uid=uid, + language=argd['ln'], + req=req, + lastupdated=__lastupdated__, + navmenuid='youralerts') + + # register event in webstat + alert_str = "%s (%d)" % (argd['name'], argd['idq']) + if user_info['email']: + user_str = "%s (%d)" % (user_info['email'], user_info['uid']) + else: + user_str = "" + try: + register_customevent("alerts", ["resume", alert_str, user_str]) + except: + register_exception(suffix="Do the webstat tables exists? Try with 'webstatadmin --load-config'") + + # display success + return page(title=_("Display alerts"), + body=html, + navtrail= """%(account)s""" % { + 'sitesecureurl' : CFG_SITE_SECURE_URL, + 'ln': argd['ln'], + 'account' : _("Your Account"), + }, + description=_("%s Personalize, Display alerts") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME), + keywords=_("%s, personalize") % CFG_SITE_NAME_INTL.get(argd['ln'], CFG_SITE_NAME), + uid=uid, + language=argd['ln'], + req=req, + lastupdated=__lastupdated__, + navmenuid='youralerts') + def popular(self, req, form): """ Display a list of popular alerts. diff --git a/modules/webstyle/css/invenio.css b/modules/webstyle/css/invenio.css index 2fdbeeb1f5..615d76e07f 100644 --- a/modules/webstyle/css/invenio.css +++ b/modules/webstyle/css/invenio.css @@ -2583,6 +2583,7 @@ a:hover.bibMergeImgClickable img { .youralerts_display_table { border: 1px solid #ffcc00; + min-width: 800px; } .youralerts_display_table_header td { background-color: #ffffcc; @@ -2628,13 +2629,18 @@ a:hover.bibMergeImgClickable img { vertical-align: top; border-bottom: 1px solid #ffcc00; } +.youralerts_display_table_content_container_main { + width: 100%; + border-bottom: 1px solid #CCCCCC; + padding-bottom: 10px; +} .youralerts_display_table_content_container_left { float: left; - width: 75%; + width: 50%; } .youralerts_display_table_content_container_right { float: right; - width: 25%; + width: 50%; } .youralerts_display_table_content_clear { clear: both; @@ -2673,14 +2679,18 @@ a:hover.bibMergeImgClickable img { text-align: right; white-space: nowrap; color: gray; + margin-top: 5px; + /*margin-bottom: 5px;*/ } .youralerts_display_table_content_options { position: relative; - top: 0px; - right: 0px; - margin-bottom: 5px; - font-size: 80%; - text-align: right; + /*top: 0px; + right: 0px;*/ + margin-top: 5px; + /*margin-bottom: 5px;*/ + font-size: 70%; + text-align: left; + white-space: nowrap; } .youralerts_display_table_content_options img{ vertical-align: top; @@ -2694,6 +2704,10 @@ a:hover.bibMergeImgClickable img { text-decoration: underline; color: #000; } + +.youralerts_display_table_content_inactive { + color: #CCCCCC; +} /* end of WebAlert module */ /* WebSearch module - YourSearches */ diff --git a/modules/webstyle/img/Makefile.am b/modules/webstyle/img/Makefile.am index 169dcb0655..08119f8be2 100644 --- a/modules/webstyle/img/Makefile.am +++ b/modules/webstyle/img/Makefile.am @@ -332,7 +332,9 @@ img_DATA = add-small.png \ yoursearches_search.png \ youralerts_alert.png \ youralerts_alert_delete.png \ - youralerts_alert_edit.png + youralerts_alert_edit.png \ + youralerts_alert_pause.png \ + youralerts_alert_resume.png tmpdir=$(localstatedir)/tmp diff --git a/modules/webstyle/img/youralerts_alert_pause.png b/modules/webstyle/img/youralerts_alert_pause.png new file mode 100644 index 0000000000000000000000000000000000000000..34c88d27d2a066b53a474d85e92b9352408fb389 GIT binary patch literal 1085 zcmV-D1j74?P)K~y-)m6Fd-V^ti*&%N*VwX~pZUvL5}SYbj)1YI}*nGi=p zOavtkBoM-e9qK<|bmMMJCK+`hK_g=sp|uoQ5Ec^DL{t_mF$0k>Fr~vNG=(zIzW3Vq zj*Fql1c>@s-tX^ot{5F1{V5y{w*r6+g+i@8Jw2-6*h)-IO%(tjlgVr%o6YizFGullbsZ&a z+s44afCnVGw6sK$jEsNHVzJ0*Y?9y^b{?fjlBSuXh(@C^ zLOPx1?CdNrVs>_xdwY8v8yj;0KmmYZ80hTmglU@4bsckabJ%{EL|4}v$eB49hJowt zZNjqilqYHc0DL|lq?G9D>Ov?O0tx6Ig`J%p00Opc<5E8P?AFa({+Pcm{Y(JJn>iMW z7RAWONU^fAk|ZxIlu}}McUKM%4{HDblANBNW;&fdcW@??VIq;>Q{5cLAxYY{%}6A2 znmieu;P~ALEz9BuAHL7=yAgK0-v0S%0ZFFQX{J&srcx=IR-Rki8?$)pX)`H*ff9hDrB`n*5<2VSE`C;2Ou3UMJ zDv?OQb=`C7+_>>J%6w&*o}Nb72*O|HM|DlLd#Mj3`S|FN0Px$=B9D(B^Y*8=J{GZ9 ztXNZ1qphy4qNAe&wY9YuPP_~$CAPM z$t33I=h4v6fad1ra}D?R_mR)%vAn#D_4ReMx3{Cdz8-^vgUDvHSX^8rfGP+8go$`O zUN~#rS#r4?CnqNvkH?wGWH>W3!*}0%hn8(}E4jh@57s%An4)D_bR37VZ=<~bV4YjZ z4cfNJYi+OmWAj-o%fivo5e&mXe}Dh!dv|v?{@VQ$j_X274T1uWQ-G8bQc4tyMQBn% z2*9NakH}( z)iu?WeSLk(FbpB36#s>L_kO>rNePeAp_B&!e;G=#@<0Gy-3O%<0HV68iVFXe7aE(s zzcc<-P6z=Z1ayxsNlE~Wd~@fc^^HHAQf^~Y<1c>$uo6%`whv3o00000NkvXXu0mjf Do{s3_ literal 0 HcmV?d00001 diff --git a/modules/webstyle/img/youralerts_alert_resume.png b/modules/webstyle/img/youralerts_alert_resume.png new file mode 100644 index 0000000000000000000000000000000000000000..7734298e153cc5c2b7ab307faf9f90ec893946dd GIT binary patch literal 1011 zcmV|L5dQQj=y)n^n+4lTC+}g7ij9OK*mu z66C(c>7^p-+{9f7cC}6mz33KBQ!Qju7pkrUElLKPxoD-CD}~VZ@xDoFusd?Ul8Wz=1iN-Ro=~5yKTYYa5N0-a5w>QFbhMItqEJHZRCr^_gN$o0q1<*c6Q_)R5`)I zIWAqkgf~yWB^-K#PtSh*)zZ(g--yWGaEEX>T@H2L{X&v{(cM${P*)VH#ZTB#n9c|jlRCV1CyRV|2d96eiV0- z{~#0$p{uJ4aH_!T^Wpf@ z$4?Fp4V`>xXy_c{oC~U|qP4XZZnyh?9rlRJ1wjyCwc22142&@_X2tgQ7Oq~s0!`Om ze&)op*H4`qE8iqJ3Hx9;LvSlV7~o;%;#> hThnAH5KJBB`!9fbDi1czEiV87002ovPDHLkV1i#zz4QP8 literal 0 HcmV?d00001 From 637bf88dc0a75f4e5980df5479d824c66021766c Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Mon, 17 Mar 2014 23:17:13 +0100 Subject: [PATCH 28/59] Webalert & Websearch: prettify display list header * Prettifies the display list header for Your Alerts and Your Searches by adding the paging navigation information. --- modules/webalert/lib/webalert_templates.py | 22 ++++++++++---------- modules/websearch/lib/websearch_templates.py | 22 ++++++++++---------- modules/webstyle/css/invenio.css | 18 +++++++++------- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/modules/webalert/lib/webalert_templates.py b/modules/webalert/lib/webalert_templates.py index ff398282bc..087f55e847 100644 --- a/modules/webalert/lib/webalert_templates.py +++ b/modules/webalert/lib/webalert_templates.py @@ -534,42 +534,42 @@ def tmpl_youralerts_display(self, 'warning_label_is_active_p' : not alert_active_p and '[ %s ] ' % _('paused') or '', } - footer = '' + paging_navigation_html = '' if paging_navigation[0]: - footer += """""" % \ + paging_navigation_html += """""" % \ (CFG_SITE_SECURE_URL, 1, step, idq, ln, '/img/sb.gif') if paging_navigation[1]: - footer += """""" % \ + paging_navigation_html += """""" % \ (CFG_SITE_SECURE_URL, page - 1, step, idq, ln, '/img/sp.gif') - footer += " " + paging_navigation_html += " " displayed_alerts_from = ((page - 1) * step) + 1 displayed_alerts_to = paging_navigation[2] and (page * step) or nb_alerts - footer += _('Displaying alerts %i to %i from %i total alerts') % \ + paging_navigation_html += _('Displaying alerts %i to %i from %i total alerts') % \ (displayed_alerts_from, displayed_alerts_to, nb_alerts) - footer += " " + paging_navigation_html += " " if paging_navigation[2]: - footer += """""" % \ + paging_navigation_html += """""" % \ (CFG_SITE_SECURE_URL, page + 1, step, idq, ln, '/img/sn.gif') if paging_navigation[3]: - footer += """""" % \ + paging_navigation_html += """""" % \ (CFG_SITE_SECURE_URL, paging_navigation[3], step, idq, ln, '/img/se.gif') out += """ - + - + %(youralerts_display_html)s -
    %(paging_navigation_html)s
    %(footer)s%(paging_navigation_html)s
    """ % {'footer': footer, +""" % {'paging_navigation_html': paging_navigation_html, 'youralerts_display_html': youralerts_display_html} return out diff --git a/modules/websearch/lib/websearch_templates.py b/modules/websearch/lib/websearch_templates.py index 8a9bfa715b..6f61373dd7 100644 --- a/modules/websearch/lib/websearch_templates.py +++ b/modules/websearch/lib/websearch_templates.py @@ -5219,42 +5219,42 @@ def tmpl_yoursearches_display(self, 'search_query_last_performed': search_query_last_performed, 'search_query_options': search_query_options} - footer = '' + paging_navigation_html = '' if paging_navigation[0]: - footer += """""" % \ + paging_navigation_html += """""" % \ (CFG_SITE_SECURE_URL, 1, step, cgi.escape(p), ln, '/img/sb.gif') if paging_navigation[1]: - footer += """""" % \ + paging_navigation_html += """""" % \ (CFG_SITE_SECURE_URL, page - 1, step, cgi.escape(p), ln, '/img/sp.gif') - footer += " " + paging_navigation_html += " " displayed_searches_from = ((page - 1) * step) + 1 displayed_searches_to = paging_navigation[2] and (page * step) or nb_queries_distinct - footer += _('Displaying searches %i to %i from %i total unique searches') % \ + paging_navigation_html += _('Displaying searches %i to %i from %i total unique searches') % \ (displayed_searches_from, displayed_searches_to, nb_queries_distinct) - footer += " " + paging_navigation_html += " " if paging_navigation[2]: - footer += """""" % \ + paging_navigation_html += """""" % \ (CFG_SITE_SECURE_URL, page + 1, step, cgi.escape(p), ln, '/img/sn.gif') if paging_navigation[3]: - footer += """""" % \ + paging_navigation_html += """""" % \ (CFG_SITE_SECURE_URL, paging_navigation[3], step, cgi.escape(p), ln, '/img/se.gif') out += """ - + - + %(yoursearches)s -
    %(paging_navigation_html)s
    %(footer)s%(paging_navigation_html)s
    """ % {'footer': footer, +""" % {'paging_navigation_html': paging_navigation_html, 'yoursearches': yoursearches} return out diff --git a/modules/webstyle/css/invenio.css b/modules/webstyle/css/invenio.css index 615d76e07f..d8dea42720 100644 --- a/modules/webstyle/css/invenio.css +++ b/modules/webstyle/css/invenio.css @@ -2587,7 +2587,11 @@ a:hover.bibMergeImgClickable img { } .youralerts_display_table_header td { background-color: #ffffcc; + color: black; padding: 2px; + font-size: 80%; + text-align: center; + white-space: nowrap; border-bottom: 1px solid #ffcc00; } .youralerts_display_table_footer td { @@ -2598,7 +2602,7 @@ a:hover.bibMergeImgClickable img { text-align: center; white-space: nowrap; } -.youralerts_display_table_footer img { +.youralerts_display_table_footer img, .youralerts_display_table_header img { border: none; margin: 0px 3px; vertical-align: bottom; @@ -2717,13 +2721,11 @@ a:hover.bibMergeImgClickable img { .websearch_yoursearches_table_header td { background-color: #ffffcc; color: black; - padding: 2px; - font-weight: bold; - font-size: 75%; - text-transform: uppercase; - border-bottom: 1px solid #ffcc00; + padding: 5px; + font-size: 80%; + text-align: center; white-space: nowrap; - vertical-align: top; + border-bottom: 1px solid #ffcc00; } .websearch_yoursearches_table_footer td { background-color: #ffffcc; @@ -2733,7 +2735,7 @@ a:hover.bibMergeImgClickable img { text-align: center; white-space: nowrap; } -.websearch_yoursearches_table_footer img { +.websearch_yoursearches_table_footer img, .websearch_yoursearches_table_header img { border: none; margin: 0px 3px; vertical-align: bottom; From c507475667706bb8b32c9d8091f9b48b5f4516b5 Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Mon, 17 Mar 2014 23:51:26 +0100 Subject: [PATCH 29/59] Webalert: smart display of popular alerts link * Hides the link to the popular alerts if there are none already defined. --- modules/webalert/lib/webalert.py | 31 +++++++++++----- modules/webalert/lib/webalert_templates.py | 42 +++++++++++++++------- 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/modules/webalert/lib/webalert.py b/modules/webalert/lib/webalert.py index 2037e1cff6..244a25559e 100644 --- a/modules/webalert/lib/webalert.py +++ b/modules/webalert/lib/webalert.py @@ -352,15 +352,28 @@ def perform_request_youralerts_display(uid, alerts = [] paging_navigation = () - out = webalert_templates.tmpl_youralerts_display(ln=ln, - alerts=alerts, - nb_alerts=nb_alerts, - nb_queries=nb_queries, - idq=idq, - page=page, - step=step, - paging_navigation=paging_navigation, - p=p) + # check if there are any popular alerts already defined + query_popular_alerts_p = """ SELECT COUNT(q.id) + FROM query q + WHERE q.type='p'""" + result_popular_alerts_p = run_sql(query_popular_alerts_p) + if result_popular_alerts_p[0][0] > 0: + popular_alerts_p = True + else: + popular_alerts_p = False + + out = webalert_templates.tmpl_youralerts_display( + ln = ln, + alerts = alerts, + nb_alerts = nb_alerts, + nb_queries = nb_queries, + idq = idq, + page = page, + step = step, + paging_navigation = paging_navigation, + p = p, + popular_alerts_p = popular_alerts_p) + return out def perform_remove_alert(alert_name, id_query, id_basket, uid, ln=CFG_SITE_LANG): diff --git a/modules/webalert/lib/webalert_templates.py b/modules/webalert/lib/webalert_templates.py index 087f55e847..e9c1b7ff78 100644 --- a/modules/webalert/lib/webalert_templates.py +++ b/modules/webalert/lib/webalert_templates.py @@ -286,7 +286,8 @@ def tmpl_youralerts_display(self, page, step, paging_navigation, - p): + p, + popular_alerts_p): """ Displays an HTML formatted list of the user alerts. If the user has specified a query id, only the user alerts based on that @@ -312,6 +313,21 @@ def tmpl_youralerts_display(self, @param idq: The specified query id for which to display the user alerts @type idq: int + + @param page: the page to be displayed + @type page: int + + @param step: the number of alerts to display per page + @type step: int + + @param paging_navigation: values to help display the paging navigation arrows + @type paging_navigation: tuple + + @param p: the search term (searching in alerts) + @type p: string + + @param popular_alerts_p: are there any popular alerts already defined? + @type popular_alerts_p: boolean """ # load the right message language @@ -330,17 +346,17 @@ def tmpl_youralerts_display(self, else: msg = _('The selected search query seems to be invalid.') msg += "
    " - msg += _('You may define new alert based on %(yoursearches)s, the %(popular_alerts)s or just by %(search_interface)s.') % \ + msg += _('You may define new alert based on %(yoursearches)s%(popular_alerts)s or just by %(search_interface)s.') % \ {'yoursearches': '%s' % (CFG_SITE_SECURE_URL, ln, _('your searches')), - 'popular_alerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('popular alerts')), + 'popular_alerts': popular_alerts_p and ', %s' % (CFG_SITE_SECURE_URL, ln, _('the popular alerts')) or '', 'search_interface': '%s' %(CFG_SITE_URL, ln, _('searching for something new'))} elif p and not idq: msg = _('You have not defined any alerts yet including the terms %s.') % \ ('' + cgi.escape(p) + '',) msg += "
    " - msg += _('You may define new alert based on %(yoursearches)s, the %(popular_alerts)s or just by %(search_interface)s.') % \ + msg += _('You may define new alert based on %(yoursearches)s%(popular_alerts)s or just by %(search_interface)s.') % \ {'yoursearches': '%s' % (CFG_SITE_SECURE_URL, ln, _('your searches')), - 'popular_alerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('popular alerts')), + 'popular_alerts': popular_alerts_p and ', %s' % (CFG_SITE_SECURE_URL, ln, _('the popular alerts')) or '', 'search_interface': '%s' %(CFG_SITE_URL, ln, _('searching for something new'))} elif p and idq: if nb_queries: @@ -353,16 +369,16 @@ def tmpl_youralerts_display(self, else: msg = _('The selected search query seems to be invalid.') msg += "
    " - msg += _('You may define new alert based on %(yoursearches)s, the %(popular_alerts)s or just by %(search_interface)s.') % \ + msg += _('You may define new alert based on %(yoursearches)s%(popular_alerts)s or just by %(search_interface)s.') % \ {'yoursearches': '%s' % (CFG_SITE_SECURE_URL, ln, _('your searches')), - 'popular_alerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('popular alerts')), + 'popular_alerts': popular_alerts_p and ', %s' % (CFG_SITE_SECURE_URL, ln, _('the popular alerts')) or '', 'search_interface': '%s' %(CFG_SITE_URL, ln, _('searching for something new'))} else: msg = _('You have not defined any alerts yet.') msg += '
    ' - msg += _('You may define new alert based on %(yoursearches)s, the %(popular_alerts)s or just by %(search_interface)s.') % \ + msg += _('You may define new alert based on %(yoursearches)s%(popular_alerts)s or just by %(search_interface)s.') % \ {'yoursearches': '%s' % (CFG_SITE_SECURE_URL, ln, _('your searches')), - 'popular_alerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('popular alerts')), + 'popular_alerts': popular_alerts_p and ', %s' % (CFG_SITE_SECURE_URL, ln, _('the popular alerts')) or '', 'search_interface': '%s' %(CFG_SITE_URL, ln, _('searching for something new'))} out = '

    ' + msg + '

    ' return out @@ -380,9 +396,9 @@ def tmpl_youralerts_display(self, {'p': '' + cgi.escape(p) + '', 'number_of_alerts': '' + str(nb_alerts) + ''} msg += '
    ' - msg += _('You may define new alert based on %(yoursearches)s, the %(popular_alerts)s or just by %(search_interface)s.') % \ + msg += _('You may define new alert based on %(yoursearches)s%(popular_alerts)s or just by %(search_interface)s.') % \ {'yoursearches': '%s' % (CFG_SITE_SECURE_URL, ln, _('your searches')), - 'popular_alerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('popular alerts')), + 'popular_alerts': popular_alerts_p and ', %s' % (CFG_SITE_SECURE_URL, ln, _('the popular alerts')) or '', 'search_interface': '%s' %(CFG_SITE_URL, ln, _('searching for something new'))} elif idq and p: msg = _('You have defined %(number_of_alerts)s alerts based on that search query including the terms %(p)s.') % \ @@ -396,9 +412,9 @@ def tmpl_youralerts_display(self, msg = _('You have defined a total of %(number_of_alerts)s alerts.') % \ {'number_of_alerts': '' + str(nb_alerts) + ''} msg += '
    ' - msg += _('You may define new alerts based on %(yoursearches)s, the %(popular_alerts)s or just by %(search_interface)s.') % \ + msg += _('You may define new alerts based on %(yoursearches)s%(popular_alerts)s or just by %(search_interface)s.') % \ {'yoursearches': '%s' % (CFG_SITE_SECURE_URL, ln, _('your searches')), - 'popular_alerts': '%s' % (CFG_SITE_SECURE_URL, ln, _('popular alerts')), + 'popular_alerts': popular_alerts_p and ', %s' % (CFG_SITE_SECURE_URL, ln, _('the popular alerts')) or '', 'search_interface': '%s' %(CFG_SITE_URL, ln, _('searching for something new'))} out = '

    ' + msg + '

    ' From 02045b1305c23e7c60245a4bbc932f4fd2100d91 Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Thu, 20 Mar 2014 01:57:25 +0100 Subject: [PATCH 30/59] Webalert: improve store results in basket menu * Improves the appearence of the "Store results in basket?" menu. * Links the basket name to the basket display when displaying alerts. --- modules/webalert/lib/webalert.py | 16 ++--- modules/webalert/lib/webalert_templates.py | 65 +++++++++++++++++++- modules/webbasket/lib/webbasket.py | 17 ----- modules/webbasket/lib/webbasket_templates.py | 16 ----- 4 files changed, 72 insertions(+), 42 deletions(-) diff --git a/modules/webalert/lib/webalert.py b/modules/webalert/lib/webalert.py index 244a25559e..b2983fa3a2 100644 --- a/modules/webalert/lib/webalert.py +++ b/modules/webalert/lib/webalert.py @@ -28,8 +28,9 @@ from invenio.webuser import isGuestUser from invenio.errorlib import register_exception from invenio.webaccount import warning_guest_user -from invenio.webbasket import create_personal_baskets_selection_box -from invenio.webbasket_dblayer import check_user_owns_baskets +from invenio.webbasket_dblayer import \ + check_user_owns_baskets, \ + get_all_user_personal_basket_ids_by_topic from invenio.messages import gettext_set_language from invenio.dateutils import convert_datestruct_to_datetext @@ -96,7 +97,7 @@ def perform_input_alert(action, id_basket, uid, is_active, - old_id_basket=None, + old_id_basket = None, ln = CFG_SITE_LANG): """get the alert settings input: action="add" for a new alert (blank form), action="modify" for an update @@ -123,10 +124,11 @@ def perform_input_alert(action, except: urlargs = "UNKNOWN" - baskets = create_personal_baskets_selection_box(uid=uid, - html_select_box_name='idb', - selected_bskid=old_id_basket, - ln=ln) + baskets = webalert_templates.tmpl_personal_basket_select_element( + bskid = old_id_basket, + personal_baskets_list = get_all_user_personal_basket_ids_by_topic(uid), + select_element_name = "idb", + ln = ln) return webalert_templates.tmpl_input_alert( ln = ln, diff --git a/modules/webalert/lib/webalert_templates.py b/modules/webalert/lib/webalert_templates.py index e9c1b7ff78..74cc123141 100644 --- a/modules/webalert/lib/webalert_templates.py +++ b/modules/webalert/lib/webalert_templates.py @@ -36,6 +36,8 @@ from invenio.urlutils import create_html_link from invenio.search_engine import guess_primary_collection_of_a_record, get_coll_ancestors from invenio.dateutils import convert_datetext_to_datestruct +from invenio.webbasket_dblayer import get_basket_ids_and_names +from invenio.webbasket_config import CFG_WEBBASKET_CATEGORIES class Template: def tmpl_errorMsg(self, ln, error_msg, rest = ""): @@ -458,8 +460,14 @@ def tmpl_youralerts_display(self, alert_frequency == 'month' and '' + _('monthly') + '') alert_details_notification = alert_notification == 'y' and _('You are notified by e-mail') or \ alert_notification == 'n' and '' - alert_details_basket = alert_basket_name and _('The results are automatically added to your basket:') + \ - ' ' + '' + cgi.escape(alert_basket_name) + '' or '' + alert_details_basket = alert_basket_name and '%s %s' % ( + _('The results are automatically added to your personal basket:'), + CFG_SITE_SECURE_URL, + CFG_WEBBASKET_CATEGORIES['PRIVATE'], + str(alert_basket_id), + ln, + cgi.escape(alert_basket_name)) \ + or '' alert_details_frequency_notification_basket = alert_details_frequency + \ (alert_details_notification and \ ' / ' + \ @@ -821,6 +829,59 @@ def tmpl_youralerts_popular(self, return out + def tmpl_personal_basket_select_element( + self, + bskid, + personal_baskets_list, + select_element_name, + ln): + """ + Returns an HTML select element with the user's personal baskets as the list of options. + """ + + _ = gettext_set_language(ln) + + out = """ + """ + + return out + def get_html_user_friendly_alert_query_args(args, ln=CFG_SITE_LANG): """ diff --git a/modules/webbasket/lib/webbasket.py b/modules/webbasket/lib/webbasket.py index 0a193eabb2..73710571b5 100644 --- a/modules/webbasket/lib/webbasket.py +++ b/modules/webbasket/lib/webbasket.py @@ -2394,23 +2394,6 @@ def create_guest_warning_box(ln=CFG_SITE_LANG): """return a warning message about logging into system""" return webbasket_templates.tmpl_create_guest_warning_box(ln) -def create_personal_baskets_selection_box(uid, - html_select_box_name='baskets', - selected_bskid=None, - ln=CFG_SITE_LANG): - """Return HTML box for basket selection. Only for personal baskets. - @param uid: user id - @param html_select_box_name: name used in html form - @param selected_bskid: basket currently selected - @param ln: language - """ - baskets = db.get_all_personal_baskets_names(uid) - return webbasket_templates.tmpl_personal_baskets_selection_box( - baskets, - html_select_box_name, - selected_bskid, - ln) - def create_basket_navtrail(uid, category=CFG_WEBBASKET_CATEGORIES['PRIVATE'], topic="", group=0, diff --git a/modules/webbasket/lib/webbasket_templates.py b/modules/webbasket/lib/webbasket_templates.py index 096d9fffba..77d4c290a5 100644 --- a/modules/webbasket/lib/webbasket_templates.py +++ b/modules/webbasket/lib/webbasket_templates.py @@ -2116,22 +2116,6 @@ def tmpl_add_group(self, bskid, selected_topic, groups=[], ln=CFG_SITE_LANG): 'submit_label': _("Add group")} return out - def tmpl_personal_baskets_selection_box(self, - baskets=[], - select_box_name='baskets', - selected_bskid=None, - ln=CFG_SITE_LANG): - """return an HTML popupmenu - @param baskets: list of (bskid, bsk_name, bsk_topic) tuples - @param select_box_name: name that will be used for the control - @param selected_bskid: id of the selcte basket, use None for no selection - @param ln: language""" - _ = gettext_set_language(ln) - elements = [(0, '- ' + _("no basket") + ' -')] - for (bskid, bsk_name, bsk_topic) in baskets: - elements.append((bskid, bsk_topic + ' > ' + bsk_name)) - return self.__create_select_menu(select_box_name, elements, selected_bskid) - def tmpl_create_guest_warning_box(self, ln=CFG_SITE_LANG): """return html warning box for non registered users""" _ = gettext_set_language(ln) From 9c505e09ab963e64d4b9e424b500932d68f56151 Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Thu, 20 Mar 2014 16:32:05 +0100 Subject: [PATCH 31/59] Webalert & Websearch: harmonize display * Harmonizes and prettifies the display of "Your Searches" and "Your Alerts" --- modules/webalert/lib/webalert_templates.py | 60 ++++++++++---------- modules/websearch/lib/websearch_templates.py | 38 +++++++------ modules/webstyle/css/invenio.css | 42 +++++--------- 3 files changed, 68 insertions(+), 72 deletions(-) diff --git a/modules/webalert/lib/webalert_templates.py b/modules/webalert/lib/webalert_templates.py index 74cc123141..d6183c50b7 100644 --- a/modules/webalert/lib/webalert_templates.py +++ b/modules/webalert/lib/webalert_templates.py @@ -461,7 +461,7 @@ def tmpl_youralerts_display(self, alert_details_notification = alert_notification == 'y' and _('You are notified by e-mail') or \ alert_notification == 'n' and '' alert_details_basket = alert_basket_name and '%s %s' % ( - _('The results are automatically added to your personal basket:'), + _('The results are added to your basket:'), CFG_SITE_SECURE_URL, CFG_WEBBASKET_CATEGORIES['PRIVATE'], str(alert_basket_id), @@ -470,21 +470,21 @@ def tmpl_youralerts_display(self, or '' alert_details_frequency_notification_basket = alert_details_frequency + \ (alert_details_notification and \ - ' / ' + \ + ' · ' + \ alert_details_notification) + \ (alert_details_basket and \ - ' / ' + \ + ' · ' + \ alert_details_basket) alert_details_search_query = get_html_user_friendly_alert_query_args(alert_query_args, ln) alert_details_creation_date = get_html_user_friendly_date_from_datetext(alert_creation_date, True, False, ln) alert_details_last_run_date = get_html_user_friendly_date_from_datetext(alert_last_run_date, True, False, ln) - alert_details_creation_last_run_dates = _('Created:') + ' ' + \ - alert_details_creation_date + \ - ' / ' + \ - _('Last run:') + ' ' + \ - alert_details_last_run_date + alert_details_creation_last_run_dates = _('Last run:') + ' ' + \ + alert_details_last_run_date + \ + ' · ' + \ + _('Created:') + ' ' + \ + alert_details_creation_date alert_details_options_pause_or_resume = create_html_link('%s/youralerts/%s' % \ (CFG_SITE_SECURE_URL, alert_active_p and 'pause' or 'resume'), @@ -520,11 +520,11 @@ def tmpl_youralerts_display(self, alert_details_options = '' % \ (CFG_SITE_URL, alert_active_p and 'pause' or 'resume') + \ alert_details_options_pause_or_resume + \ - ' · ' + \ + ' · ' + \ ' ' % \ (CFG_SITE_URL,) + \ alert_details_options_edit + \ - ' · ' + \ + ' · ' + \ '' % \ (CFG_SITE_URL,) + \ alert_details_options_delete @@ -536,19 +536,15 @@ def tmpl_youralerts_display(self,
    -
    %(warning_label_is_active_p)s%(alert_name)s
    -
    %(alert_details_frequency_notification_basket)s
    +
    %(warning_label_is_active_p)s%(alert_name_label)s %(alert_name)s
    %(alert_details_search_query)s
    +
    %(alert_details_frequency_notification_basket)s
    -
    -
    -
    %(alert_details_options)s
    -
    -
    -
    %(alert_details_creation_last_run_dates)s
    -
    +
    %(alert_details_creation_last_run_dates)s
    +
    %(alert_details_options)s
    """ % {'counter': counter, + 'alert_name_label' : _('Alert'), 'alert_name': cgi.escape(alert_name), 'alert_details_frequency_notification_basket': alert_details_frequency_notification_basket, 'alert_details_search_query': alert_details_search_query, @@ -999,7 +995,7 @@ def get_html_user_friendly_date_from_datetext(given_date, if days_old == 0: out = _('Today') elif days_old < 7: - out = str(days_old) + ' ' + _('day(s) ago') + out = str(days_old) + ' ' + _('days ago') elif days_old == 7: out = _('A week ago') elif days_old < 14: @@ -1010,21 +1006,27 @@ def get_html_user_friendly_date_from_datetext(given_date, out = _('More than two weeks ago') elif days_old == 30: out = _('A month ago') - elif days_old < 180: + elif days_old < 90: out = _('More than a month ago') + elif days_old < 180: + out = _('More than three months ago') elif days_old < 365: out = _('More than six months ago') - else: + elif days_old < 730: out = _('More than a year ago') + elif days_old < 1095: + out = _('More than two years ago') + elif days_old < 1460: + out = _('More than three years ago') + elif days_old < 1825: + out = _('More than four years ago') + else: + out = _('More than five years ago') if show_full_date: - out += '' + \ - ' ' + _('on') + ' ' + \ - given_date.split()[0] + '' + out += ' ' + _('on') + ' ' + given_date.split()[0] if show_full_time: - out += '' + \ - ' ' + _('at') + ' ' + \ - given_date.split()[1] + '' + out += ' ' + _('at') + ' ' + given_date.split()[1] else: - out = _('Unknown') + out = _('unknown') return out diff --git a/modules/websearch/lib/websearch_templates.py b/modules/websearch/lib/websearch_templates.py index 6f61373dd7..d85b3b498a 100644 --- a/modules/websearch/lib/websearch_templates.py +++ b/modules/websearch/lib/websearch_templates.py @@ -5193,14 +5193,14 @@ def tmpl_yoursearches_display(self, search_query_options_alert = search_query_number_of_user_alerts and \ """%s""" % \ - (CFG_SITE_SECURE_URL, ln, search_query_id, CFG_SITE_URL, _('Edit your existing alert(s)')) + \ - '   ' + \ + (CFG_SITE_SECURE_URL, ln, search_query_id, CFG_SITE_URL, _('Edit your existing alerts')) + \ + ' · ' + \ """%s""" % \ (CFG_SITE_SECURE_URL, ln, search_query_id, CFG_SITE_URL, _('Set up as a new alert')) or \ """%s""" % \ (CFG_SITE_SECURE_URL, ln, search_query_id, CFG_SITE_URL, _('Set up as a new alert')) - search_query_options = "%s   %s" % \ + search_query_options = "%s · %s" % \ (search_query_options_search, \ search_query_options_alert) @@ -5211,7 +5211,7 @@ def tmpl_yoursearches_display(self,
    %(search_query_details)s
    -
    %(search_query_last_performed)s
    +
    %(search_query_last_performed)s
    %(search_query_options)s
    """ % {'counter': counter, @@ -5283,9 +5283,9 @@ def get_html_user_friendly_search_query_args(args, args_dict = parse_qs(args) if not args_dict.has_key('p') and not args_dict.has_key('p1') and not args_dict.has_key('p2') and not args_dict.has_key('p3'): - search_patterns_html = _('Search for everything') + search_patterns_html = _('You searched for everything') else: - search_patterns_html = _('Search for') + ' ' + search_patterns_html = _('You searched for') + ' ' if args_dict.has_key('p'): search_patterns_html += '' + cgi.escape(args_dict['p'][0]) + '' if args_dict.has_key('f'): @@ -5376,7 +5376,7 @@ def get_html_user_friendly_date_from_datetext(given_date, if days_old == 0: out = _('Today') elif days_old < 7: - out = str(days_old) + ' ' + _('day(s) ago') + out = str(days_old) + ' ' + _('days ago') elif days_old == 7: out = _('A week ago') elif days_old < 14: @@ -5387,21 +5387,27 @@ def get_html_user_friendly_date_from_datetext(given_date, out = _('More than two weeks ago') elif days_old == 30: out = _('A month ago') - elif days_old < 180: + elif days_old < 90: out = _('More than a month ago') + elif days_old < 180: + out = _('More than three months ago') elif days_old < 365: out = _('More than six months ago') - else: + elif days_old < 730: out = _('More than a year ago') + elif days_old < 1095: + out = _('More than two years ago') + elif days_old < 1460: + out = _('More than three years ago') + elif days_old < 1825: + out = _('More than four years ago') + else: + out = _('More than five years ago') if show_full_date: - out += '' + \ - ' ' + _('on') + ' ' + \ - given_date.split()[0] + '' + out += ' ' + _('on') + ' ' + given_date.split()[0] if show_full_time: - out += '' + \ - ' ' + _('at') + ' ' + \ - given_date.split()[1] + '' + out += ' ' + _('at') + ' ' + given_date.split()[1] else: - out = _('Unknown') + out = _('unknown') return out diff --git a/modules/webstyle/css/invenio.css b/modules/webstyle/css/invenio.css index d8dea42720..2849d124bb 100644 --- a/modules/webstyle/css/invenio.css +++ b/modules/webstyle/css/invenio.css @@ -2635,19 +2635,6 @@ a:hover.bibMergeImgClickable img { } .youralerts_display_table_content_container_main { width: 100%; - border-bottom: 1px solid #CCCCCC; - padding-bottom: 10px; -} -.youralerts_display_table_content_container_left { - float: left; - width: 50%; -} -.youralerts_display_table_content_container_right { - float: right; - width: 50%; -} -.youralerts_display_table_content_clear { - clear: both; } .youralerts_display_table_counter { background-color: white; @@ -2660,40 +2647,33 @@ a:hover.bibMergeImgClickable img { } .youralerts_display_table_content_name { margin-bottom: 5px; - font-weight: bold; position: relative; left: 0px; top: 0px; } .youralerts_display_table_content_details { margin-bottom: 5px; - font-size: 80%; position: relative; left: 0px; } .youralerts_display_table_content_search_query { margin-bottom: 5px; - font-size: 80%; position: relative; left: 0px; } .youralerts_display_table_content_dates { + margin-bottom: 5px; position: relative; - font-size: 70%; - text-align: right; - white-space: nowrap; + left: 0px; + font-size: 80%; color: gray; - margin-top: 5px; - /*margin-bottom: 5px;*/ + white-space: nowrap; } .youralerts_display_table_content_options { + margin-bottom: 5px; position: relative; - /*top: 0px; - right: 0px;*/ - margin-top: 5px; - /*margin-bottom: 5px;*/ - font-size: 70%; - text-align: left; + left: 0px; + font-size: 80%; white-space: nowrap; } .youralerts_display_table_content_options img{ @@ -2712,6 +2692,9 @@ a:hover.bibMergeImgClickable img { .youralerts_display_table_content_inactive { color: #CCCCCC; } +.youralerts_display_table_content_inactive a, .youralerts_display_table_content_inactive a:link, .youralerts_display_table_content_inactive a:visited, .youralerts_display_table_content_inactive a:active, .youralerts_display_table_content_inactive a:hover { + color: #CCCCCC; +} /* end of WebAlert module */ /* WebSearch module - YourSearches */ @@ -2775,6 +2758,11 @@ a:hover.bibMergeImgClickable img { color: grey; border-bottom: 1px solid #ffcc00; } +.websearch_yoursearches_table_content_dates { + color: gray; + margin: 5px; + font-size: 80%; +} .websearch_yoursearches_table_content_options { margin: 5px; font-size: 80%; From 42d60dee8a4c589bbd17ad79e59713d1643394d3 Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Mon, 24 Mar 2014 19:55:04 +0100 Subject: [PATCH 32/59] Websession: Revamp the "Your Account" page * Revamps the "Your Account" page by arranging items to be displayed in a 3-column grid, trying to keep all columns equal in size. * Re-arranges code for consistency and readability. --- modules/webalert/lib/webalert.py | 42 +-- modules/webalert/lib/webalert_templates.py | 52 +-- modules/webbasket/lib/webbasket.py | 31 +- modules/webbasket/lib/webbasket_dblayer.py | 49 ++- modules/webbasket/lib/webbasket_templates.py | 28 ++ modules/webcomment/lib/webcomment.py | 17 + .../webcomment/lib/webcomment_templates.py | 17 + modules/webmessage/lib/webmessage.py | 19 +- .../webmessage/lib/webmessage_templates.py | 33 +- modules/websearch/lib/websearch_templates.py | 24 +- .../websearch/lib/websearch_yoursearches.py | 35 +- modules/websession/lib/webaccount.py | 160 +++++---- modules/websession/lib/webgroup.py | 26 +- .../websession/lib/websession_templates.py | 307 ++++++++++++------ .../websession/lib/websession_webinterface.py | 80 ++--- modules/webstyle/css/invenio.css | 66 ++-- modules/websubmit/lib/websubmit_templates.py | 17 + 17 files changed, 606 insertions(+), 397 deletions(-) diff --git a/modules/webalert/lib/webalert.py b/modules/webalert/lib/webalert.py index b2983fa3a2..0e8b76387a 100644 --- a/modules/webalert/lib/webalert.py +++ b/modules/webalert/lib/webalert.py @@ -27,7 +27,6 @@ from invenio.dbquery import run_sql from invenio.webuser import isGuestUser from invenio.errorlib import register_exception -from invenio.webaccount import warning_guest_user from invenio.webbasket_dblayer import \ check_user_owns_baskets, \ get_all_user_personal_basket_ids_by_topic @@ -141,10 +140,7 @@ def perform_input_alert(action, old_id_basket = old_id_basket, id_basket = id_basket, id_query = id_query, - is_active = is_active, - guest = isGuestUser(uid), - guesttxt = warning_guest_user(type="alerts", ln=ln) - ) + is_active = is_active) def check_alert_is_unique(id_basket, id_query, uid, ln=CFG_SITE_LANG ): """check the user does not have another alert for the specified query and basket""" @@ -566,30 +562,22 @@ def is_selected(var, fld): else: return "" -def account_list_alerts(uid, ln=CFG_SITE_LANG): - """account_list_alerts: list alert for the account page +def account_user_alerts(uid, ln=CFG_SITE_LANG): + """ + Information on the user's alerts for the "Your Account" page. input: the user id language - output: the list of alerts Web page""" - query = """ SELECT q.id, q.urlargs, a.id_user, a.id_query, - a.id_basket, a.alert_name, a.frequency, - a.notification, - DATE_FORMAT(a.date_creation,'%%d %%b %%Y'), - DATE_FORMAT(a.date_lastrun,'%%d %%b %%Y'), - a.id_basket - FROM query q, user_query_basket a - WHERE a.id_user=%s AND a.id_query=q.id - ORDER BY a.alert_name ASC """ - res = run_sql(query, (uid,)) - alerts = [] - if len(res): - for row in res: - alerts.append({ - 'id' : row[0], - 'name' : row[5] - }) + output: the information in HTML + """ + + query = """ SELECT COUNT(*) + FROM user_query_basket + WHERE id_user = %s""" + params = (uid,) + result = run_sql(query, params) + alerts = result[0][0] - return webalert_templates.tmpl_account_list_alerts(ln=ln, alerts=alerts) + return webalert_templates.tmpl_account_user_alerts(alerts, ln) def perform_request_youralerts_popular(ln=CFG_SITE_LANG): """ @@ -636,7 +624,7 @@ def count_user_alerts_for_given_query(id_user, """ query = """ SELECT COUNT(id_query) - FROM user_query_basket AS uqb + FROM user_query_basket WHERE id_user=%s AND id_query=%s""" params = (id_user, id_query) diff --git a/modules/webalert/lib/webalert_templates.py b/modules/webalert/lib/webalert_templates.py index d6183c50b7..507f276ac7 100644 --- a/modules/webalert/lib/webalert_templates.py +++ b/modules/webalert/lib/webalert_templates.py @@ -99,35 +99,30 @@ def tmpl_textual_query_info_from_urlargs(self, ln, args): out += "" + _("Collection") + ": " + "; ".join(args['cc']) + "
    " return out - def tmpl_account_list_alerts(self, ln, alerts): + def tmpl_account_user_alerts(self, alerts, ln = CFG_SITE_LANG): """ - Displays all the alerts in the main "Your account" page + Information on the user's alerts for the "Your Account" page. Parameters: - + - 'alerts' *int* - The number of alerts - 'ln' *string* - The language to display the interface in - - - 'alerts' *array* - The existing alerts IDs ('id' + 'name' pairs) """ - # load the right message language _ = gettext_set_language(ln) - out = """
    - %(you_own)s: - -   -
    """ % { - 'show' : _("SHOW"), - } + if alerts > 0: + out = _('You have defined a total of %(x_url_open)s%(x_alerts_number)s alerts%(x_url_close)s.') % \ + {'x_url_open' : '' % (CFG_SITE_SECURE_URL, ln), + 'x_alerts_number': str(alerts), + 'x_url_close' : ''} + else: + out = _('You have not defined any alerts yet.') + + out += " " + _('You may define new alerts based on %(x_yoursearches_url_open)syour searches%(x_url_close)s or just by %(x_search_interface_url_open)ssearching for something new%(x_url_close)s.') % \ + {'x_yoursearches_url_open' : '' % (CFG_SITE_SECURE_URL, ln), + 'x_search_interface_url_open' : '' % (CFG_SITE_URL, ln), + 'x_url_close' : '',} + return out def tmpl_input_alert(self, @@ -141,9 +136,7 @@ def tmpl_input_alert(self, old_id_basket, id_basket, id_query, - is_active, - guest, - guesttxt): + is_active): """ Displays an alert adding form. @@ -170,10 +163,6 @@ def tmpl_input_alert(self, - 'id_query' *string* - The id of the query associated to this alert - 'is_active' *boolean* - is the alert active or not - - - 'guest' *bool* - If the user is a guest user - - - 'guesttxt' *string* - The HTML content of the warning box for guest users (produced by webaccount.tmpl_warning_guest_user) """ # load the right message language @@ -274,9 +263,6 @@ def tmpl_input_alert(self, out += '' % old_id_basket out += "" - if guest: - out += guesttxt - return out def tmpl_youralerts_display(self, @@ -776,10 +762,6 @@ def tmpl_youralerts_popular(self, - 'args' *string* - The query string - 'textargs' *string* - The textual description of the query string - - - 'guest' *bool* - If the user is a guest user - - - 'guesttxt' *string* - The HTML content of the warning box for guest users (produced by webaccount.tmpl_warning_guest_user) """ # load the right message language diff --git a/modules/webbasket/lib/webbasket.py b/modules/webbasket/lib/webbasket.py index 73710571b5..8d39f1045b 100644 --- a/modules/webbasket/lib/webbasket.py +++ b/modules/webbasket/lib/webbasket.py @@ -2573,30 +2573,15 @@ def create_webbasket_navtrail(uid, return out -def account_list_baskets(uid, ln=CFG_SITE_LANG): - """Display baskets informations on account page""" - _ = gettext_set_language(ln) +def account_user_baskets(uid, ln=CFG_SITE_LANG): + """ + Display baskets informations on account page + """ + (personal, group, external) = db.count_baskets(uid) - link = '%s' - base_url = CFG_SITE_URL + '/yourbaskets/display?category=%s&ln=' + ln - personal_text = personal - if personal: - url = base_url % CFG_WEBBASKET_CATEGORIES['PRIVATE'] - personal_text = link % (url, personal_text) - group_text = group - if group: - url = base_url % CFG_WEBBASKET_CATEGORIES['GROUP'] - group_text = link % (url, group_text) - external_text = external - if external: - url = base_url % CFG_WEBBASKET_CATEGORIES['EXTERNAL'] - else: - url = CFG_SITE_URL + '/yourbaskets/list_public_baskets?ln=' + ln - external_text = link % (url, external_text) - out = _("You have %(x_nb_perso)s personal baskets and are subscribed to %(x_nb_group)s group baskets and %(x_nb_public)s other users public baskets.") %\ - {'x_nb_perso': personal_text, - 'x_nb_group': group_text, - 'x_nb_public': external_text} + + out = webbasket_templates.tmpl_account_user_baskets(personal, group, external, ln) + return out def page_start(req, of='xm'): diff --git a/modules/webbasket/lib/webbasket_dblayer.py b/modules/webbasket/lib/webbasket_dblayer.py index 4dd7f37949..d7fc581a67 100644 --- a/modules/webbasket/lib/webbasket_dblayer.py +++ b/modules/webbasket/lib/webbasket_dblayer.py @@ -127,23 +127,39 @@ ########################## General functions ################################## def count_baskets(uid): - """Return (nb personal baskets, nb group baskets, nb external - baskets) tuple for given user""" - query1 = "SELECT COUNT(id) FROM bskBASKET WHERE id_owner=%s" - res1 = run_sql(query1, (int(uid),)) - personal = __wash_sql_count(res1) - query2 = """SELECT count(ugbsk.id_bskbasket) - FROM usergroup_bskBASKET ugbsk LEFT JOIN user_usergroup uug - ON ugbsk.id_usergroup=uug.id_usergroup - WHERE uug.id_user=%s AND uug.user_status!=%s - GROUP BY ugbsk.id_usergroup""" - params = (int(uid), CFG_WEBSESSION_USERGROUP_STATUS['PENDING']) - res2 = run_sql(query2, params) - if len(res2): - groups = reduce(lambda x, y: x + y, map(lambda x: x[0], res2)) + """ + Return (number of personal baskets, + number of group baskets, + number of external baskets) + tuple for the given user based on their user id. + """ + + # TODO: Maybe put this in a try..except ? + uid = int(uid) + + query_personal = """SELECT COUNT(id) + FROM bskBASKET + WHERE id_owner=%s""" + params_personal = (uid,) + res_personal = run_sql(query_personal, params_personal) + personal = __wash_sql_count(res_personal) + + query_group = """ SELECT COUNT(ugbsk.id_bskbasket) + FROM usergroup_bskBASKET ugbsk + LEFT JOIN user_usergroup uug + ON ugbsk.id_usergroup = uug.id_usergroup + WHERE uug.id_user = %s + AND uug.user_status != %s + GROUP BY ugbsk.id_usergroup""" + params_group = (uid, CFG_WEBSESSION_USERGROUP_STATUS['PENDING']) + res_group = run_sql(query_group, params_group) + if len(res_group): + groups = reduce(lambda x, y: x + y, map(lambda x: x[0], res_group)) else: groups = 0 + external = count_external_baskets(uid) + return (personal, groups, external) def check_user_owns_baskets(uid, bskids): @@ -1567,13 +1583,16 @@ def get_all_external_basket_ids_and_names(uid): def count_external_baskets(uid): """Returns the number of external baskets the user is subscribed to.""" + # TODO: Maybe put this in a try..except ? + uid = int(uid) + query = """ SELECT COUNT(ubsk.id_bskBASKET) FROM user_bskBASKET ubsk LEFT JOIN bskBASKET bsk ON (bsk.id=ubsk.id_bskBASKET AND ubsk.id_user=%s) WHERE bsk.id_owner!=%s""" - params = (int(uid), int(uid)) + params = (uid, uid) res = run_sql(query, params) diff --git a/modules/webbasket/lib/webbasket_templates.py b/modules/webbasket/lib/webbasket_templates.py index 77d4c290a5..3d82ad8b5f 100644 --- a/modules/webbasket/lib/webbasket_templates.py +++ b/modules/webbasket/lib/webbasket_templates.py @@ -4185,6 +4185,34 @@ def tmpl_create_export_as_list(self, return out + def tmpl_account_user_baskets(self, personal, group, external, ln = CFG_SITE_LANG): + """ + Information on the user's baskets for the "Your Account" page. + """ + + _ = gettext_set_language(ln) + + if (personal, group, external) == (0, 0, 0): + out = _("You have not created any personal baskets yet, you are not part of any group baskets and you have not subscribed to any public baskets.") + else: + x_generic_url_open = '' % (CFG_SITE_SECURE_URL, ln) + out = _("You have %(x_personal_url_open)s%(x_personal_nb)s personal baskets%(x_personal_url_close)s, you are part of %(x_group_url_open)s%(x_group_nb)s group baskets%(x_group_url_close)s and you are subscribed to %(x_public_url_open)s%(x_public_nb)s public baskets%(x_public_url_close)s.") % \ + {'x_personal_url_open' : '%s' % (personal > 0 and x_generic_url_open % (CFG_WEBBASKET_CATEGORIES['PRIVATE'],) or '',), + 'x_personal_nb' : str(personal), + 'x_personal_url_close' : '%s' % (personal > 0 and '' or '',), + 'x_group_url_open' : '%s' % (group > 0 and x_generic_url_open % (CFG_WEBBASKET_CATEGORIES['GROUP'],) or '',), + 'x_group_nb' : str(group), + 'x_group_url_close' : '%s' % (group > 0 and '' or '',), + 'x_public_url_open' : '%s' % (external > 0 and x_generic_url_open % (CFG_WEBBASKET_CATEGORIES['EXTERNAL'],) or '',), + 'x_public_nb' : str(external), + 'x_public_url_close' : '%s' % (external > 0 and '' or '',),} + + out += " " + _("You might be interested in looking through %(x_url_open)sall the public baskets%(x_url_close)s.") % \ + {'x_url_open' : '' % (CFG_SITE_SECURE_URL, ln), + 'x_url_close' : '',} + + return out + ############################################# ########## SUPPLEMENTARY FUNCTIONS ########## ############################################# diff --git a/modules/webcomment/lib/webcomment.py b/modules/webcomment/lib/webcomment.py index b3f1fabbda..8a40013333 100644 --- a/modules/webcomment/lib/webcomment.py +++ b/modules/webcomment/lib/webcomment.py @@ -2037,3 +2037,20 @@ def perform_display_your_comments(user_info, nb_total_results=nb_total_results, nb_total_pages=nb_total_pages, ln=ln) + +def account_user_comments(uid, ln = CFG_SITE_LANG): + """ + Information on the user comments for the "Your Account" page. + """ + + query = """ SELECT COUNT(id) + FROM cmtRECORDCOMMENT + WHERE id_user = %s + AND star_score = 0""" + params = (uid,) + result = run_sql(query, params) + comments = result[0][0] + + out = webcomment_templates.tmpl_account_user_comments(comments, ln) + + return out diff --git a/modules/webcomment/lib/webcomment_templates.py b/modules/webcomment/lib/webcomment_templates.py index d47089e80c..a22000ce66 100644 --- a/modules/webcomment/lib/webcomment_templates.py +++ b/modules/webcomment/lib/webcomment_templates.py @@ -2612,3 +2612,20 @@ def tmpl_your_comments(self, user_info, comments, page_number=1, selected_order_ out += '
    ' return out + + def tmpl_account_user_comments(self, comments, ln = CFG_SITE_LANG): + """ + Information on the user's comments for the "Your Account" page. + """ + + _ = gettext_set_language(ln) + + if comments > 0: + out = _("You have submitted %(x_url_open)s%(x_comments)s comments%(x_url_close)s so far.") % \ + {'x_url_open' : '' % (CFG_SITE_SECURE_URL, ln), + 'x_comments' : str(comments), + 'x_url_close' : '',} + else: + out = _("You have not yet submitted any comment. Browse documents from the search interface and take part to discussions!") + + return out diff --git a/modules/webmessage/lib/webmessage.py b/modules/webmessage/lib/webmessage.py index 40d3a93673..1e5cd9f2f6 100644 --- a/modules/webmessage/lib/webmessage.py +++ b/modules/webmessage/lib/webmessage.py @@ -458,17 +458,20 @@ def listing(name1, name2): title = _("Your Messages") return (body, title, get_navtrail(ln)) -def account_new_mail(uid, ln=CFG_SITE_LANG): +def account_user_messages(uid, ln = CFG_SITE_LANG): """ - display new mail info for myaccount.py page. + Information on the user's messages for the "Your Account" page @param uid: user id (int) - @param ln: language - @return: html body + @param ln: interface language (str) + @return: information in HTML """ - nb_new_mail = db.get_nb_new_messages_for_user(uid) - total_mail = db.get_nb_readable_messages_for_user(uid) - return webmessage_templates.tmpl_account_new_mail(nb_new_mail, - total_mail, ln) + + total = db.get_nb_readable_messages_for_user(uid) + unread = db.get_nb_new_messages_for_user(uid) + + out = webmessage_templates.tmpl_account_user_messages(total, unread, ln) + + return out def get_navtrail(ln=CFG_SITE_LANG, title=""): """ diff --git a/modules/webmessage/lib/webmessage_templates.py b/modules/webmessage/lib/webmessage_templates.py index 35ff83843d..1f442344b9 100644 --- a/modules/webmessage/lib/webmessage_templates.py +++ b/modules/webmessage/lib/webmessage_templates.py @@ -35,7 +35,7 @@ create_year_selectbox from invenio.urlutils import create_html_link, create_url from invenio.htmlutils import escape_html -from invenio.config import CFG_SITE_URL, CFG_SITE_LANG +from invenio.config import CFG_SITE_URL, CFG_SITE_SECURE_URL, CFG_SITE_LANG from invenio.messages import gettext_set_language from invenio.webuser import get_user_info @@ -681,21 +681,28 @@ def tmpl_user_or_group_search(self, 'add_button' : add_button} return out - def tmpl_account_new_mail(self, nb_new_mail=0, total_mail=0, ln=CFG_SITE_LANG): + def tmpl_account_user_messages(self, total = 0, unread = 0, ln = CFG_SITE_LANG): """ - display infos about inbox (used by myaccount.py) - @param nb_new_mail: number of new mails + Information about the user's messages for the "Your Account" page. + @param total: total number of messages + @param unread: number of unread messages @param ln: language - return: html output. + return: information in HTML """ + _ = gettext_set_language(ln) - out = _("You have %(x_nb_new)s new messages out of %(x_nb_total)s messages") % \ - {'x_nb_new': '' + str(nb_new_mail) + '', - 'x_nb_total': create_html_link(CFG_SITE_URL + '/yourmessages/', - {'ln': ln}, - str(total_mail), - {}, - False, False)} - return out + '.' + if total > 0: + out = _("You have received a total of %(x_url_open)s%(x_total)s messages (%(x_unread)s unread)%(x_url_close)s.") % \ + {'x_url_open' : '' % (CFG_SITE_SECURE_URL, ln), + 'x_total' : str(total), + 'x_unread' : str(unread), + 'x_url_close' : ''} + else: + out = _("You have not received any messages yet.") + + out += " " + _("You might be interested in %(x_url_open)swriting a new message%(x_url_close)s.") % \ + {'x_url_open' : '' % (CFG_SITE_SECURE_URL, ln), + 'x_url_close' : '',} + return out diff --git a/modules/websearch/lib/websearch_templates.py b/modules/websearch/lib/websearch_templates.py index d85b3b498a..1c82a761ab 100644 --- a/modules/websearch/lib/websearch_templates.py +++ b/modules/websearch/lib/websearch_templates.py @@ -5126,10 +5126,6 @@ def tmpl_yoursearches_display(self, @param guest: Whether the user is a guest or not @type guest: boolean - - @param guesttxt: The HTML content of the warning box for guest users - (produced by webaccount.tmpl_warning_guest_user) - @type guesttxt: string """ # Load the right language @@ -5259,6 +5255,26 @@ def tmpl_yoursearches_display(self, return out + def tmpl_account_user_searches(self, unique, total, ln = CFG_SITE_LANG): + """ + Information on the user's searches for the "Your Account" page + """ + + _ = gettext_set_language(ln) + + if unique > 0: + out = _("You have performed %(x_url_open)s%(unique)s unique searches%(x_url_close)s in a total of %(total)s searches.") % \ + {'x_url_open' : '' % (CFG_SITE_SECURE_URL, ln), + 'unique' : str(unique), + 'x_url_close' : '', + 'total' : str(total),} + else: + out = _("You have not searched for anything yet. You may want to start by the %(x_url_open)ssearch interface%(x_url_close)s first.") % \ + {'x_url_open' : '' % (CFG_SITE_SECURE_URL, ln), + 'x_url_close' : ''} + + return out + def get_html_user_friendly_search_query_args(args, ln=CFG_SITE_LANG): """ diff --git a/modules/websearch/lib/websearch_yoursearches.py b/modules/websearch/lib/websearch_yoursearches.py index ddb74a6566..25c621d0e7 100644 --- a/modules/websearch/lib/websearch_yoursearches.py +++ b/modules/websearch/lib/websearch_yoursearches.py @@ -21,7 +21,6 @@ from invenio.config import CFG_SITE_LANG, CFG_SITE_SECURE_URL from invenio.dbquery import run_sql -from invenio.webaccount import warning_guest_user from invenio.messages import gettext_set_language from invenio.webuser import isGuestUser from urllib import quote @@ -150,31 +149,27 @@ def perform_request_yoursearches_display(uid, p=p, ln = ln) -def account_list_searches(uid, - ln=CFG_SITE_LANG): +def account_user_searches(uid, ln = CFG_SITE_LANG): """ - Display a short summary of the searches the user has performed. + Information on the user's searches for the "Your Account" page @param uid: The user id @type uid: int - @return: A short summary of the user searches. + @param ln: The interface language + @type ln: str + @return: A short summary on the user's searches. """ - # load the right language - _ = gettext_set_language(ln) - - query = """ SELECT COUNT(uq.id_query) - FROM user_query uq - WHERE uq.id_user=%s""" + # Calculate the number of total and unique queries + query = """ SELECT COUNT(id_query), + COUNT(DISTINCT(id_query)) + FROM user_query + WHERE id_user=%s""" params = (uid,) - result = run_sql(query, params, 1) - if result: - nb_queries_total = result[0][0] - else: - nb_queries_total = 0 + res = run_sql(query, params) + + total = res[0][0] + unique = res[0][1] - out = _("You have made %(x_nb)s queries. A %(x_url_open)sdetailed list%(x_url_close)s is available with a possibility to (a) view search results and (b) subscribe to an automatic email alerting service for these queries.") % \ - {'x_nb': nb_queries_total, - 'x_url_open': '' % (CFG_SITE_SECURE_URL, ln), - 'x_url_close': ''} + out = websearch_templates.tmpl_account_user_searches(unique, total, ln) return out diff --git a/modules/websession/lib/webaccount.py b/modules/websession/lib/webaccount.py index 6e98dc14bc..bc4fca0403 100644 --- a/modules/websession/lib/webaccount.py +++ b/modules/websession/lib/webaccount.py @@ -47,6 +47,16 @@ from invenio import web_api_key +from invenio.webbasket import account_user_baskets +from invenio.webalert import account_user_alerts +from invenio.websearch_yoursearches import account_user_searches +from invenio.webmessage import account_user_messages +from invenio.webgroup import account_user_groups +from invenio.websubmit_templates import account_user_submissions +from invenio.websession_templates import account_user_approvals +from invenio.webcomment import account_user_comments +from invenio.websession_templates import account_user_tickets + def perform_info(req, ln): """Display the main features of CDS personalize""" uid = getUid(req) @@ -74,15 +84,19 @@ def perform_display_external_user_settings(settings, ln): html_settings += websession_templates.tmpl_external_setting(ln, key, value) return print_settings and websession_templates.tmpl_external_user_settings(ln, html_settings) or "" -def perform_youradminactivities(user_info, ln): - """Return text for the `Your Admin Activities' box. Analyze - whether user UID has some admin roles, and if yes, then print - suitable links for the actions he can do. If he's not admin, - print a simple non-authorized message.""" +def account_user_administration(user_info, ln = CFG_SITE_LANG): + """ + Information for for the "Your Admin Activities" box. + Analyzes whether user UID has some admin roles, and if yes, then returns + suitable links for the action they can do. If they are not admin, return + a simple non-authorized message. + """ + your_role_actions = acc_find_user_role_actions(user_info) your_roles = [] your_admin_activities = [] guest = int(user_info['guest']) + for (role, action) in your_role_actions: if role not in your_roles: your_roles.append(role) @@ -90,48 +104,68 @@ def perform_youradminactivities(user_info, ln): your_admin_activities.append(action) if SUPERADMINROLE in your_roles: - for action in ("runbibedit", "cfgbibformat", "cfgoaiharvest", "cfgoairepository", "cfgbibrank", "cfgbibindex", "cfgwebaccess", "cfgwebcomment", "cfgwebsearch", "cfgwebsubmiit", "cfgbibknowledge", "runbatchuploader"): + for action in ("runbibedit", + "cfgbibformat", + "cfgoaiharvest", + "cfgoairepository", + "cfgbibrank", + "cfgbibindex", + "cfgwebaccess", + "cfgwebcomment", + "cfgwebsearch", + "cfgwebsubmiit", + "cfgbibknowledge", + "runbatchuploader"): if action not in your_admin_activities: your_admin_activities.append(action) - return websession_templates.tmpl_account_adminactivities( - ln = ln, - uid = user_info['uid'], - guest = guest, - roles = your_roles, - activities = your_admin_activities, - ) + out = websession_templates.tmpl_account_adminactivities( + ln = ln, + uid = user_info['uid'], + guest = guest, + roles = your_roles, + activities = your_admin_activities) -def perform_display_account(req, username, bask, aler, sear, msgs, loan, grps, sbms, appr, admn, ln, comments): - """Display a dynamic page that shows the user's account.""" + return out - # load the right message language - _ = gettext_set_language(ln) +def perform_display_account( + uid, + user_info, + user_name, + user_baskets_p, + user_alerts_p, + user_searches_p, + user_messages_p, + user_loans_p, + user_groups_p, + user_submissions_p, + user_approvals_p, + user_administration_p, + user_comments_p, + user_tickets_p, + ln): + """ + Display a dynamic page that shows the user's account. + """ - uid = getUid(req) - user_info = collect_user_info(req) - #your account - if int(user_info['guest']): - user = "guest" - login = "%s/youraccount/login?ln=%s" % (CFG_SITE_SECURE_URL, ln) - accBody = _("You are logged in as guest. You may want to %(x_url_open)slogin%(x_url_close)s as a regular user.") %\ - {'x_url_open': '', - 'x_url_close': ''} - accBody += "

    " - bask=aler=msgs=comments= _("The %(x_fmt_open)sguest%(x_fmt_close)s users need to %(x_url_open)sregister%(x_url_close)s first") %\ - {'x_fmt_open': '', - 'x_fmt_close': '', - 'x_url_open': '', - 'x_url_close': ''} - sear= _("No queries found") - else: - user = username - accBody = websession_templates.tmpl_account_body( - ln = ln, - user = user, - ) + # Load the right message language + _ = gettext_set_language(ln) - #Display warnings if user is superuser + user_account = websession_templates.tmpl_account_body(ln = ln, user = user_name) + user_baskets = user_baskets_p and account_user_baskets(uid, ln) or None + user_alerts = user_alerts_p and account_user_alerts(uid, ln) or None + user_searches = user_searches_p and account_user_searches(uid, ln) or None + user_messages = user_messages_p and account_user_messages(uid, ln) or None + # TODO: Write a function that returns "Loans" information for the user + user_loans = user_loans_p and None or None + user_groups = user_groups_p and account_user_groups(uid, ln) or None + user_submissions = user_submissions_p and account_user_submissions(ln) or None + user_approvals = user_approvals_p and account_user_approvals(ln) or None + user_administration = user_administration_p and account_user_administration(user_info, ln) or None + user_comments = user_comments_p and account_user_comments(uid, ln) or None + user_tickets = user_tickets_p and account_user_tickets(ln) or None + + # Display warnings if user is superuser roles = acc_find_user_role_actions(user_info) warnings = "0" warning_list = [] @@ -150,26 +184,22 @@ def perform_display_account(req, username, bask, aler, sear, msgs, loan, grps, s warnings = "1" warning_list.append(email_autogenerated_warning) - #check if tickets ok - tickets = (acc_authorize_action(user_info, 'runbibedit')[0] == 0) - return websession_templates.tmpl_account_page( - ln = ln, - warnings = warnings, - warning_list = warning_list, - accBody = accBody, - baskets = bask, - alerts = aler, - searches = sear, - messages = msgs, - loans = loan, - groups = grps, - submissions = sbms, - approvals = appr, - tickets = tickets, - administrative = admn, - comments = comments, - ) + ln = ln, + warnings = warnings, + warning_list = warning_list, + account = user_account, + baskets = user_baskets, + alerts = user_alerts, + searches = user_searches, + messages = user_messages, + loans = user_loans, + groups = user_groups, + submissions = user_submissions, + approvals = user_approvals, + tickets = user_tickets, + administrative = user_administration, + comments = user_comments) def superuser_account_warnings(): """Check to see whether admin accounts have default / blank password etc. Returns a list""" @@ -258,18 +288,6 @@ def template_account(title, body, ln): body = body ) -def warning_guest_user(type, ln=CFG_SITE_LANG): - """It returns an alert message,showing that the user is a guest user and should log into the system.""" - - # load the right message language - _ = gettext_set_language(ln) - - return websession_templates.tmpl_warning_guest_user( - ln = ln, - type = type, - ) - - def perform_delete(ln): """Delete the account of the user, not implement yet.""" # TODO diff --git a/modules/websession/lib/webgroup.py b/modules/websession/lib/webgroup.py index e3105113b6..99a2e780b6 100644 --- a/modules/websession/lib/webgroup.py +++ b/modules/websession/lib/webgroup.py @@ -761,21 +761,31 @@ def perform_request_reject_member(uid, ln=ln) return body -def account_group(uid, ln=CFG_SITE_LANG): - """Display group info for myaccount.py page. +def account_user_groups(uid, ln=CFG_SITE_LANG): + """ + Information on the user's groups for the "Your Account" page @param uid: user id (int) - @param ln: language - @return: html body + @param ln: language (str) + @return: information in HTML """ - nb_admin_groups = db.count_nb_group_user(uid, + + nb_admin_groups = db.count_nb_group_user( + uid, CFG_WEBSESSION_USERGROUP_STATUS["ADMIN"]) - nb_member_groups = db.count_nb_group_user(uid, + + nb_member_groups = db.count_nb_group_user( + uid, CFG_WEBSESSION_USERGROUP_STATUS["MEMBER"]) + nb_total_groups = nb_admin_groups + nb_member_groups - return websession_templates.tmpl_group_info(nb_admin_groups, + + out = websession_templates.tmpl_account_user_groups( + nb_admin_groups, nb_member_groups, nb_total_groups, - ln=ln) + ln = ln) + + return out def get_navtrail(ln=CFG_SITE_LANG, title=""): """Gets the navtrail for title. diff --git a/modules/websession/lib/websession_templates.py b/modules/websession/lib/websession_templates.py index 91d89f4711..cf09bb43f2 100644 --- a/modules/websession/lib/websession_templates.py +++ b/modules/websession/lib/websession_templates.py @@ -686,14 +686,14 @@ def tmpl_account_body(self, ln, user): # load the right message language _ = gettext_set_language(ln) - out = _("You are logged in as %(x_user)s. You may want to a) %(x_url1_open)slogout%(x_url1_close)s; b) edit your %(x_url2_open)saccount settings%(x_url2_close)s.") %\ - {'x_user': user, - 'x_url1_open': '', + out = _("You are logged in as %(x_user)s. You may want to %(x_url1_open)sedit your account settings%(x_url1_close)s or %(x_url2_open)slogout%(x_url2_close)s.") %\ + {'x_user': "" + cgi.escape(user) + "", + 'x_url1_open': '', 'x_url1_close': '', - 'x_url2_open': '', - 'x_url2_close': '', - } - return out + "

    " + 'x_url2_open': '', + 'x_url2_close': '',} + + return out def tmpl_account_template(self, title, body, ln, url): """ @@ -710,18 +710,32 @@ def tmpl_account_template(self, title, body, ln, url): - 'url' *string* - The URL to go to the proper section """ - out =""" - - - - - - - -
    %s
    %s
    """ % (url, title, body) + out = """ +
    +
    %s
    +
    %s
    +
    + """ % (url, title, body) + return out - def tmpl_account_page(self, ln, warnings, warning_list, accBody, baskets, alerts, searches, messages, loans, groups, submissions, approvals, tickets, administrative, comments): + def tmpl_account_page( + self, + ln, + warnings, + warning_list, + account, + baskets, + alerts, + searches, + messages, + loans, + groups, + submissions, + approvals, + tickets, + administrative, + comments): """ Displays the your account page @@ -729,7 +743,7 @@ def tmpl_account_page(self, ln, warnings, warning_list, accBody, baskets, alerts - 'ln' *string* - The language to display the interface in - - 'accBody' *string* - The body of the heading block + - 'account' *string* - The body of the heading block - 'baskets' *string* - The body of the baskets block @@ -759,53 +773,88 @@ def tmpl_account_page(self, ln, warnings, warning_list, accBody, baskets, alerts if warnings == "1": out += self.tmpl_general_warnings(warning_list) - out += self.tmpl_account_template(_("Your Account"), accBody, ln, '/youraccount/edit?ln=%s' % ln) - if messages: - out += self.tmpl_account_template(_("Your Messages"), messages, ln, '/yourmessages/display?ln=%s' % ln) + # Store all the information to be displayed in the items list. + # Only store items that should be displayed. + # NOTE: The order in which we append items to the list matters! + items = [] + account is not None and items.append(self.tmpl_account_template( + _("Your Account"), account, ln, "/youraccount/edit?ln=%s" % ln)) + groups is not None and items.append(self.tmpl_account_template( + _("Your Groups"), groups, ln, "/yourgroups/display?ln=%s" % ln)) + administrative is not None and items.append(self.tmpl_account_template( + _("Your Administrative Activities"), administrative, ln, "/admin")) + messages is not None and items.append(self.tmpl_account_template( + _("Your Messages"), messages, ln, "/yourmessages/display?ln=%s" % ln)) + searches is not None and items.append(self.tmpl_account_template( + _("Your Searches"), searches, ln, "/yoursearches/display?ln=%s" % ln)) + alerts is not None and items.append(self.tmpl_account_template( + _("Your Alerts"), alerts, ln, "/youralerts/display?ln=%s" % ln)) + baskets is not None and items.append(self.tmpl_account_template( + _("Your Baskets"), baskets, ln, "/yourbaskets/display?ln=%s" % ln)) + comments is not None and items.append(self.tmpl_account_template( + _("Your Comments"), comments, ln, "/yourcomments/?ln=%s" % ln)) + submissions is not None and items.append(self.tmpl_account_template( + _("Your Submissions"), submissions, ln, "/yoursubmissions.py?ln=%s" % ln)) + approvals is not None and items.append(self.tmpl_account_template( + _("Your Approvals"), approvals, ln, "/yourapprovals.py?ln=%s" % ln)) + loans is not None and items.append(self.tmpl_account_template( + _("Your Loans"), loans, ln, "/yourloans/display?ln=%s" % ln)) + tickets is not None and items.append(self.tmpl_account_template( + _("Your Tickets"), tickets, ln, "/yourtickets?ln=%s" % ln)) + + # Prepare 3 more lists, 1 for each of the 3 columns + items_in_column_1_of_3 = [] + items_in_column_2_of_3 = [] + items_in_column_3_of_3 = [] + + # Sort-of-intelligently arrange the items in the 3 lists + while items: + # While there are still items to display, get the first item, + # removing it from the list of items to display. + item = items.pop(0) + # The following "1-liner" does the following (>= Python 2.5): + # * For each of the 3 lists (1 for each column) + # calculate the sum of the length of its items (see the lambda). + # The lenght of an itme is the literal length of the string + # to be displayed, which includes HTML code. This of course is + # not the most accurate way, but a good approximation and also + # very efficient. + # * Pick the list that has the smallest sum. + # * Append the current item to that list. + min(items_in_column_1_of_3, + items_in_column_2_of_3, + items_in_column_3_of_3, + key = lambda l: sum([len(i) for i in l]) + ).append(item) + + # Finally, create the HTML code for the entire grid. + out += """ +
    +
    """ - if loans: - out += self.tmpl_account_template(_("Your Loans"), loans, ln, '/yourloans/display?ln=%s' % ln) + for item_in_column_1_of_3 in items_in_column_1_of_3: + out += item_in_column_1_of_3 - if baskets: - out += self.tmpl_account_template(_("Your Baskets"), baskets, ln, '/yourbaskets/display?ln=%s' % ln) + out += """ +
    +
    """ - if comments: - comments_description = _("You can consult the list of %(x_url_open)syour comments%(x_url_close)s submitted so far.") - comments_description %= {'x_url_open': '', - 'x_url_close': ''} - out += self.tmpl_account_template(_("Your Comments"), comments_description, ln, '/yourcomments/?ln=%s' % ln) - if alerts: - out += self.tmpl_account_template(_("Your Alert Searches"), alerts, ln, '/youralerts/display?ln=%s' % ln) + for item_in_column_2_of_3 in items_in_column_2_of_3: + out += item_in_column_2_of_3 - if searches: - out += self.tmpl_account_template(_("Your Searches"), searches, ln, '/yoursearches/display?ln=%s' % ln) + out += """ +
    +
    + """ + + for item_in_column_3_of_3 in items_in_column_3_of_3: + out += item_in_column_3_of_3 + + out += """ +
    +
    + """ - if groups: - groups_description = _("You can consult the list of %(x_url_open)syour groups%(x_url_close)s you are administering or are a member of.") - groups_description %= {'x_url_open': '', - 'x_url_close': ''} - out += self.tmpl_account_template(_("Your Groups"), groups_description, ln, '/yourgroups/display?ln=%s' % ln) - - if submissions: - submission_description = _("You can consult the list of %(x_url_open)syour submissions%(x_url_close)s and inquire about their status.") - submission_description %= {'x_url_open': '', - 'x_url_close': ''} - out += self.tmpl_account_template(_("Your Submissions"), submission_description, ln, '/yoursubmissions.py?ln=%s' % ln) - - if approvals: - approval_description = _("You can consult the list of %(x_url_open)syour approvals%(x_url_close)s with the documents you approved or refereed.") - approval_description %= {'x_url_open': '', - 'x_url_close': ''} - out += self.tmpl_account_template(_("Your Approvals"), approval_description, ln, '/yourapprovals.py?ln=%s' % ln) - - #check if this user might have tickets - if tickets: - ticket_description = _("You can consult the list of %(x_url_open)syour tickets%(x_url_close)s.") - ticket_description %= {'x_url_open': '', - 'x_url_close': ''} - out += self.tmpl_account_template(_("Your Tickets"), ticket_description, ln, '/yourtickets?ln=%s' % ln) - if administrative: - out += self.tmpl_account_template(_("Your Administrative Activities"), administrative, ln, '/admin') return out def tmpl_account_emailMessage(self, ln, msg): @@ -1223,25 +1272,19 @@ def tmpl_register_page(self, ln, referer, level): def tmpl_account_adminactivities(self, ln, uid, guest, roles, activities): """ - Displays the admin activities block for this user + Displays the admin activities block for this user. Parameters: - - 'ln' *string* - The language to display the interface in - - 'uid' *string* - The used id - - 'guest' *boolean* - If the user is guest - - 'roles' *array* - The current user roles - - 'activities' *array* - The user allowed activities """ # load the right message language _ = gettext_set_language(ln) - out = "" # guest condition if guest: return _("You seem to be a guest user. You have to %(x_url_open)slogin%(x_url_close)s first.") % \ @@ -1250,66 +1293,70 @@ def tmpl_account_adminactivities(self, ln, uid, guest, roles, activities): # no rights condition if not roles: - return "

    " + _("You are not authorized to access administrative functions.") + "

    " + return _("You are not authorized to access administrative functions.") # displaying form - out += "

    " + _("You are enabled to the following roles: %(x_role)s.") % {'x_role': ('' + ", ".join(roles) + "")} + '

    ' + out = _("You are enabled to the following roles: %(x_role)s.") % \ + {'x_role': ('' + ", ".join(roles) + "")} if activities: # print proposed links: activities.sort(lambda x, y: cmp(x.lower(), y.lower())) - tmp_out = '' + tmp_out = activities and '
      ' or '' for action in activities: if action == "runbibedit": - tmp_out += """
          %s""" % (CFG_SITE_URL, CFG_SITE_RECORD, _("Run Record Editor")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, CFG_SITE_RECORD, _("Run Record Editor")) if action == "runbibeditmulti": - tmp_out += """
          %s""" % (CFG_SITE_URL, CFG_SITE_RECORD, _("Run Multi-Record Editor")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, CFG_SITE_RECORD, _("Run Multi-Record Editor")) if action == "runauthorlist": - tmp_out += """
          %s""" % (CFG_SITE_URL, _("Run Author List Manager")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, _("Run Author List Manager")) if action == "runbibcirculation": - tmp_out += """
          %s""" % (CFG_SITE_URL, ln, _("Run BibCirculation")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, ln, _("Run BibCirculation")) if action == "runbibmerge": - tmp_out += """
          %s""" % (CFG_SITE_URL, CFG_SITE_RECORD, _("Run Record Merger")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, CFG_SITE_RECORD, _("Run Record Merger")) if action == "runbibswordclient": - tmp_out += """
          %s""" % (CFG_SITE_URL, CFG_SITE_RECORD, _("Run BibSword Client")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, CFG_SITE_RECORD, _("Run BibSword Client")) if action == "runbatchuploader": - tmp_out += """
          %s""" % (CFG_SITE_URL, ln, _("Run Batch Uploader")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, ln, _("Run Batch Uploader")) if action == "cfgbibformat": - tmp_out += """
          %s""" % (CFG_SITE_URL, ln, _("Configure BibFormat")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, ln, _("Configure BibFormat")) if action == "cfgbibknowledge": - tmp_out += """
          %s""" % (CFG_SITE_URL, ln, _("Configure BibKnowledge")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, ln, _("Configure BibKnowledge")) if action == "cfgoaiharvest": - tmp_out += """
          %s""" % (CFG_SITE_URL, ln, _("Configure OAI Harvest")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, ln, _("Configure OAI Harvest")) if action == "cfgoairepository": - tmp_out += """
          %s""" % (CFG_SITE_URL, ln, _("Configure OAI Repository")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, ln, _("Configure OAI Repository")) if action == "cfgbibindex": - tmp_out += """
          %s""" % (CFG_SITE_URL, ln, _("Configure BibIndex")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, ln, _("Configure BibIndex")) if action == "cfgbibrank": - tmp_out += """
          %s""" % (CFG_SITE_URL, ln, _("Configure BibRank")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, ln, _("Configure BibRank")) if action == "cfgwebaccess": - tmp_out += """
          %s""" % (CFG_SITE_URL, ln, _("Configure WebAccess")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, ln, _("Configure WebAccess")) if action == "cfgwebcomment": - tmp_out += """
          %s""" % (CFG_SITE_URL, ln, _("Configure WebComment")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, ln, _("Configure WebComment")) if action == "cfgweblinkback": - tmp_out += """
          %s""" % (CFG_SITE_URL, ln, _("Configure WebLinkback")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, ln, _("Configure WebLinkback")) if action == "cfgwebjournal": - tmp_out += """
          %s""" % (CFG_SITE_URL, ln, _("Configure WebJournal")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, ln, _("Configure WebJournal")) if action == "cfgwebsearch": - tmp_out += """
          %s""" % (CFG_SITE_URL, ln, _("Configure WebSearch")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, ln, _("Configure WebSearch")) if action == "cfgwebsubmit": - tmp_out += """
          %s""" % (CFG_SITE_URL, ln, _("Configure WebSubmit")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, ln, _("Configure WebSubmit")) if action == "runbibdocfile": - tmp_out += """
          %s""" % (CFG_SITE_URL, CFG_SITE_RECORD, ln, _("Run Document File Manager")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, CFG_SITE_RECORD, ln, _("Run Document File Manager")) if action == "cfgbibsort": - tmp_out += """
          %s""" % (CFG_SITE_URL, ln, _("Configure BibSort")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, ln, _("Configure BibSort")) if action == "runinfomanager": - tmp_out += """
          %s""" % (CFG_SITE_URL, ln, _("Run Info Space Manager")) + tmp_out += """
    • %s
      • """ % (CFG_SITE_URL, ln, _("Run Info Space Manager")) + tmp_out += activities and '
    ' or '' + if tmp_out: - out += _("Here are some interesting web admin links for you:") + tmp_out + out += "

    " + _("Here are some interesting web admin links for you:") + tmp_out + + out += _("For the complete list of administrative activities, see the %(x_url_open)sAdmin Area%(x_url_close)s.") %\ + {'x_url_open' : '' % (CFG_SITE_URL, ln), + 'x_url_close' : ''} - out += "
    " + _("For more admin-level activities, see the complete %(x_url_open)sAdmin Area%(x_url_close)s.") %\ - {'x_url_open': '', - 'x_url_close': ''} return out def tmpl_create_userinfobox(self, ln, url_referer, guest, username, submitter, referee, admin, usebaskets, usemessages, usealerts, usegroups, useloans, usestats): @@ -2595,22 +2642,40 @@ def tmpl_delete_msg(self, body += '
    ' return subject, body - def tmpl_group_info(self, nb_admin_groups=0, nb_member_groups=0, nb_total_groups=0, ln=CFG_SITE_LANG): - """ - display infos about groups (used by myaccount.py) - @param nb_admin_group: number of groups the user is admin of - @param nb_member_group: number of groups the user is member of - @param total_group: number of groups the user belongs to + def tmpl_account_user_groups( + self, + nb_admin_groups = 0, + nb_member_groups = 0, + nb_total_groups = 0, + ln = CFG_SITE_LANG): + """ + Information on the user's groups for the "Your Account" page + @param nb_admin_groups: number of groups the user is admin of + @param nb_member_groups: number of groups the user is member of + @param nb_total_groups: number of groups the user belongs to @param ln: language return: html output. """ + _ = gettext_set_language(ln) - out = _("You can consult the list of %(x_url_open)s%(x_nb_total)i groups%(x_url_close)s you are subscribed to (%(x_nb_member)i) or administering (%(x_nb_admin)i).") - out %= {'x_url_open': '', - 'x_nb_total': nb_total_groups, - 'x_url_close': '', - 'x_nb_admin': nb_admin_groups, - 'x_nb_member': nb_member_groups} + + if nb_total_groups > 0: + out = _("You are a member of %(x_url_open)s%(x_total)s groups%(x_url_close)s") % \ + {'x_url_open' : '' % (CFG_SITE_SECURE_URL, ln), + 'x_total' : str(nb_total_groups), + 'x_url_close' : '',} + if nb_admin_groups > 0: + out += ", " + _("%(x_admin)s of which you administer") % \ + {'x_admin' : str(nb_admin_groups),} + out += "." + else: + out = _("You are not member of any groups.") + + out += " " + _("You might be interested in %(x_join_url_open)sjoining a group%(x_url_close)s or %(x_create_url_open)screating a new one%(x_url_close)s.") % \ + {'x_join_url_open' : '' % (CFG_SITE_SECURE_URL, ln), + 'x_create_url_open' : '' % (CFG_SITE_SECURE_URL, ln), + 'x_url_close' : '',} + return out def tmpl_general_warnings(self, warning_list, ln=CFG_SITE_LANG): @@ -2877,3 +2942,29 @@ def construct_button(provider, size, button_class): } """ return out + +def account_user_approvals(ln = CFG_SITE_LANG): + """ + Information on the user approvals for the "Your Account" page. + """ + + _ = gettext_set_language(ln) + + out = _("You can review all the documents you approved or refereed in %(x_url_open)syour approvals%(x_url_close)s.") % \ + {'x_url_open' : '' % (CFG_SITE_SECURE_URL, ln), + 'x_url_close' : '',} + + return out + +def account_user_tickets(ln = CFG_SITE_LANG): + """ + Information on the user tickets for the "Your Account" page. + """ + + _ = gettext_set_language(ln) + + out = _("You can review %(x_url_open)syour tickets%(x_url_close)s.") % \ + {'x_url_open': '' % (CFG_SITE_SECURE_URL, ln), + 'x_url_close': '',} + + return out diff --git a/modules/websession/lib/websession_webinterface.py b/modules/websession/lib/websession_webinterface.py index be00e2aa34..73599cb9c1 100644 --- a/modules/websession/lib/websession_webinterface.py +++ b/modules/websession/lib/websession_webinterface.py @@ -44,11 +44,7 @@ from invenio import webuser from invenio.webpage import page from invenio import webaccount -from invenio import webbasket -from invenio import webalert -from invenio import websearch_yoursearches from invenio.dbquery import run_sql -from invenio.webmessage import account_new_mail from invenio.access_control_engine import acc_authorize_action from invenio.webinterface_handler import wash_urlargd, WebInterfaceDirectory from invenio import webinterface_handler_config as apache @@ -74,13 +70,10 @@ from invenio import web_api_key - import invenio.template websession_templates = invenio.template.load('websession') bibcatalog_templates = invenio.template.load('bibcatalog') - - class WebInterfaceYourAccountPages(WebInterfaceDirectory): _exports = ['', 'edit', 'change', 'lost', 'display', @@ -227,39 +220,50 @@ def display(self, req, form): navmenuid='youraccount') if webuser.isGuestUser(uid): - return page(title=_("Your Account"), - body=webaccount.perform_info(req, args['ln']), - description="%s Personalize, Main page" % CFG_SITE_NAME_INTL.get(args['ln'], CFG_SITE_NAME), - keywords=_("%s, personalize") % CFG_SITE_NAME_INTL.get(args['ln'], CFG_SITE_NAME), - uid=uid, - req=req, + # TODO: use CFG_WEBSESSION_DIFFERENTIATE_BETWEEN_GUESTS to decide whether to redirect the user to + # the login page or show them the page that is currently shown. + return page(title = _("Your Account"), + body = webaccount.perform_info(req, args['ln']), + description = "%s Personalize, Main page" % CFG_SITE_NAME_INTL.get(args['ln'], CFG_SITE_NAME), + keywords = _("%s, personalize") % CFG_SITE_NAME_INTL.get(args['ln'], CFG_SITE_NAME), + uid = uid, + req = req, secure_page_p = 1, - language=args['ln'], - lastupdated=__lastupdated__, - navmenuid='youraccount') + language = args['ln'], + lastupdated = __lastupdated__, + navmenuid = 'youraccount') - username = webuser.get_nickname_or_email(uid) + user_name = webuser.get_nickname_or_email(uid) user_info = webuser.collect_user_info(req) - bask = user_info['precached_usebaskets'] and webbasket.account_list_baskets(uid, ln=args['ln']) or '' - aler = user_info['precached_usealerts'] and webalert.account_list_alerts(uid, ln=args['ln']) or '' - sear = websearch_yoursearches.account_list_searches(uid, ln=args['ln']) - msgs = user_info['precached_usemessages'] and account_new_mail(uid, ln=args['ln']) or '' - grps = user_info['precached_usegroups'] and webgroup.account_group(uid, ln=args['ln']) or '' - appr = user_info['precached_useapprove'] - sbms = user_info['precached_viewsubmissions'] - comments = user_info['precached_sendcomments'] - loan = '' - admn = webaccount.perform_youradminactivities(user_info, args['ln']) - return page(title=_("Your Account"), - body=webaccount.perform_display_account(req, username, bask, aler, sear, msgs, loan, grps, sbms, appr, admn, args['ln'], comments), - description="%s Personalize, Main page" % CFG_SITE_NAME_INTL.get(args['ln'], CFG_SITE_NAME), - keywords=_("%s, personalize") % CFG_SITE_NAME_INTL.get(args['ln'], CFG_SITE_NAME), - uid=uid, - req=req, - secure_page_p = 1, - language=args['ln'], - lastupdated=__lastupdated__, - navmenuid='youraccount') + + body = webaccount.perform_display_account( + uid, + user_info, + user_name, + user_baskets_p = user_info['precached_usebaskets'], + user_alerts_p = user_info['precached_usealerts'], + user_searches_p = True, + user_messages_p = user_info['precached_usemessages'], + user_loans_p = False, + user_groups_p = user_info['precached_usegroups'], + user_submissions_p = user_info['precached_viewsubmissions'], + user_approvals_p = user_info['precached_useapprove'], + user_administration_p = True, + user_comments_p = user_info['precached_sendcomments'], + user_tickets_p = (acc_authorize_action(user_info, 'runbibedit')[0] == 0), + ln = args['ln']) + + return page( + title = _("Your Account"), + body = body, + description = "%s Personalize, Main page" % CFG_SITE_NAME_INTL.get(args['ln'], CFG_SITE_NAME), + keywords = _("%s, personalize") % CFG_SITE_NAME_INTL.get(args['ln'], CFG_SITE_NAME), + uid = uid, + req = req, + secure_page_p = 1, + language = args['ln'], + lastupdated = __lastupdated__, + navmenuid = 'youraccount') def apikey(self, req, form): args = wash_urlargd(form, { @@ -713,7 +717,7 @@ def youradminactivities(self, req, form): navmenuid='admin') return page(title=_("Your Administrative Activities"), - body=webaccount.perform_youradminactivities(user_info, args['ln']), + body=webaccount.account_user_administration(user_info, args['ln']), navtrail="""""" % (CFG_SITE_SECURE_URL, args['ln']) + _("Your Account") + """""", description="%s Personalize, Main page" % CFG_SITE_NAME_INTL.get(args['ln'], CFG_SITE_NAME), keywords=_("%s, personalize") % CFG_SITE_NAME_INTL.get(args['ln'], CFG_SITE_NAME), diff --git a/modules/webstyle/css/invenio.css b/modules/webstyle/css/invenio.css index 2849d124bb..791f07dc10 100644 --- a/modules/webstyle/css/invenio.css +++ b/modules/webstyle/css/invenio.css @@ -829,38 +829,50 @@ a.google:hover { text-align: left; vertical-align: top; } -.youraccountbox { - color: #000; - background: #fff; - padding: 1px; - margin: 5px 0px 5px 0px; - border-collapse: collapse; - border-top: 1px solid #fc0; + +div.youraccount_grid { + width: 100%; } -.youraccountheader { - color: #333; - background: #ffc; - font-weight: normal; - font-size: small; - vertical-align: top; - text-align: left; + +div.youraccount_grid:after { + content: ""; + display: table; + clear: both; } -.youraccountbody { - color: #333; - background: #fff; - padding: 0px 5px 0px 5px; - margin-bottom:5px; - font-size: small; - text-align: left; - vertical-align: top; + +div.youraccount_grid_column_1_3 { + width: 33.33%; + float: left; } -th.youraccountheader a:link, th.youraccountheader a:visited { - color:#000000; - text-decoration:none; + +div.youraccount_grid_column_content { + margin: 7px 7px 14px 7px; + border: 1px solid #FFCC00; + /*border-radius: 0px 0px 15px 0px;*/ } -th.youraccountheader a:hover { - text-decoration:underline; + +div.youraccount_grid_column_title { + border-bottom: 1px solid #FFCC00; + background-color: #FFFFCC; + padding: 7px; } + +div.youraccount_grid_column_title a, div.youraccount_grid_column_title a:link, div.youraccount_grid_column_title a:visited, div.youraccount_grid_column_title a:active { + color: black; + text-decoration: none; + font-weight: bold; +} + +div.youraccount_grid_column_title a:hover { + color: black; + text-decoration: underline; + font-weight: bold; +} + +div.youraccount_grid_column_body { + padding: 10px; +} + .adminbox { color: #000; background: #f1f1f1; diff --git a/modules/websubmit/lib/websubmit_templates.py b/modules/websubmit/lib/websubmit_templates.py index fec9602e68..cf018b6631 100644 --- a/modules/websubmit/lib/websubmit_templates.py +++ b/modules/websubmit/lib/websubmit_templates.py @@ -3302,3 +3302,20 @@ def displaycplxdoc_displayauthaction(action, linkText): "action" : action, "linkText" : linkText } + +def account_user_submissions(ln = CFG_SITE_LANG): + """ + Information on the user submissions for the "Your Account" page. + """ + + # TODO: give more information, such as the number of submissions the user has, + # how many are finished, pending etc. For that we need to pass the user id and + # introduce another function (in websubmit.py?) that calculates those numbers. + + _ = gettext_set_language(ln) + + out = _("You can review the status of all %(x_url_open)syour submissions%(x_url_close)s.") % \ + {'x_url_open' : '' % (CFG_SITE_SECURE_URL, ln), + 'x_url_close' : '',} + + return out From 6ff2ec666fa973832d2290331b295e530592c841 Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Wed, 26 Mar 2014 11:42:26 +0100 Subject: [PATCH 33/59] General: fix kwalitee reported issues * Fixes various warnings and errors reported by kwalitee. --- modules/webalert/lib/webalert.py | 1 - modules/webalert/lib/webalert_templates.py | 5 +- modules/webbasket/lib/webbasket.py | 35 ++-- modules/webbasket/lib/webbasket_dblayer.py | 161 ++---------------- modules/webbasket/lib/webbasket_templates.py | 21 +-- .../webbasket/lib/webbasket_webinterface.py | 2 +- modules/webcomment/lib/webcomment.py | 27 ++- modules/webmessage/lib/webmessage.py | 2 +- .../websearch/lib/websearch_yoursearches.py | 3 +- modules/websession/lib/webaccount.py | 8 +- modules/websession/lib/webgroup.py | 1 - .../websession/lib/websession_templates.py | 22 +-- .../websession/lib/websession_webinterface.py | 8 +- 13 files changed, 70 insertions(+), 226 deletions(-) diff --git a/modules/webalert/lib/webalert.py b/modules/webalert/lib/webalert.py index 0e8b76387a..5b44943144 100644 --- a/modules/webalert/lib/webalert.py +++ b/modules/webalert/lib/webalert.py @@ -25,7 +25,6 @@ from invenio.config import CFG_SITE_LANG from invenio.dbquery import run_sql -from invenio.webuser import isGuestUser from invenio.errorlib import register_exception from invenio.webbasket_dblayer import \ check_user_owns_baskets, \ diff --git a/modules/webalert/lib/webalert_templates.py b/modules/webalert/lib/webalert_templates.py index 507f276ac7..389c1472db 100644 --- a/modules/webalert/lib/webalert_templates.py +++ b/modules/webalert/lib/webalert_templates.py @@ -134,7 +134,6 @@ def tmpl_input_alert(self, notification, baskets, old_id_basket, - id_basket, id_query, is_active): """ @@ -158,8 +157,6 @@ def tmpl_input_alert(self, - 'old_id_basket' *string* - The id of the previous basket of this alert - - 'id_basket' *string* - The id of the basket of this alert - - 'id_query' *string* - The id of the query associated to this alert - 'is_active' *boolean* - is the alert active or not @@ -832,7 +829,7 @@ def tmpl_personal_basket_select_element( 'selected': bskid == '' and ' selected="selected"' or '', 'label': _("Don't store results in basket...")} - # Create the s and """ % ('* ' + _('Your personal baskets') + ' *',) diff --git a/modules/webbasket/lib/webbasket.py b/modules/webbasket/lib/webbasket.py index 8d39f1045b..fa24d325d7 100644 --- a/modules/webbasket/lib/webbasket.py +++ b/modules/webbasket/lib/webbasket.py @@ -1053,7 +1053,7 @@ def perform_request_search(uid, p="", b="", n=0, - #format='xm', + #record_format='xm', ln=CFG_SITE_LANG): """Search the baskets... @param uid: user id @@ -1160,12 +1160,12 @@ def perform_request_search(uid, # The search format for external records. This means in which format will # the external records be fetched from the database to be searched then. - format = 'xm' + record_format = 'xm' ### Calculate the search results for the user's personal baskets ### if b.startswith("P") or not b: personal_search_results = {} - personal_items = db.get_all_items_in_user_personal_baskets(uid, selected_topic, format) + personal_items = db.get_all_items_in_user_personal_baskets(uid, selected_topic, record_format) personal_local_items = personal_items[0] personal_external_items = personal_items[1] personal_external_items_xml_records = {} @@ -1239,7 +1239,7 @@ def perform_request_search(uid, ### Calculate the search results for the user's group baskets ### if b.startswith("G") or not b: group_search_results = {} - group_items = db.get_all_items_in_user_group_baskets(uid, selected_group_id, format) + group_items = db.get_all_items_in_user_group_baskets(uid, selected_group_id, record_format) group_local_items = group_items[0] group_external_items = group_items[1] group_external_items_xml_records = {} @@ -1328,7 +1328,7 @@ def perform_request_search(uid, ### Calculate the search results for the user's public baskets ### if b.startswith("E") or not b: public_search_results = {} - public_items = db.get_all_items_in_user_public_baskets(uid, format) + public_items = db.get_all_items_in_user_public_baskets(uid, record_format) public_local_items = public_items[0] public_external_items = public_items[1] public_external_items_xml_records = {} @@ -1411,7 +1411,7 @@ def perform_request_search(uid, ### Calculate the search results for all the public baskets ### if b.startswith("A"): all_public_search_results = {} - all_public_items = db.get_all_items_in_all_public_baskets(format) + all_public_items = db.get_all_items_in_all_public_baskets(record_format) all_public_local_items = all_public_items[0] all_public_external_items = all_public_items[1] all_public_external_items_xml_records = {} @@ -1695,14 +1695,15 @@ def perform_request_delete_note(uid, #warnings_notes.append('WRN_WEBBASKET_DELETE_INVALID_NOTE') warnings_html += webbasket_templates.tmpl_warnings(exc.message, ln) - (body, warnings, navtrail) = perform_request_display(uid=uid, - selected_category=category, - selected_topic=topic, - selected_group_id=group_id, - selected_bskid=bskid, - selected_recid=recid, - of='hb', - ln=CFG_SITE_LANG) + (body, dummy, navtrail) = perform_request_display( + uid=uid, + selected_category=category, + selected_topic=topic, + selected_group_id=group_id, + selected_bskid=bskid, + selected_recid=recid, + of='hb', + ln=CFG_SITE_LANG) body = warnings_html + body #warnings.extend(warnings_notes) @@ -2042,7 +2043,7 @@ def perform_request_delete(uid, bskid, confirmed=0, if not(db.check_user_owns_baskets(uid, [bskid])): try: raise InvenioWebBasketWarning(_('Sorry, you do not have sufficient rights on this basket.')) - except InvenioWebBasketWarning, exc: + except InvenioWebBasketWarning: register_exception(stream='warning') #warnings.append(exc.message) #warnings.append(('WRN_WEBBASKET_NO_RIGHTS',)) @@ -2111,7 +2112,7 @@ def perform_request_edit(uid, bskid, topic="", new_name='', if rights != CFG_WEBBASKET_SHARE_LEVELS['MANAGE']: try: raise InvenioWebBasketWarning(_('Sorry, you do not have sufficient rights on this basket.')) - except InvenioWebBasketWarning, exc: + except InvenioWebBasketWarning: register_exception(stream='warning') #warnings.append(exc.message) #warnings.append(('WRN_WEBBASKET_NO_RIGHTS',)) @@ -2470,7 +2471,7 @@ def create_basket_navtrail(uid, if bskid: basket = db.get_public_basket_infos(bskid) if basket: - basket_html = """ > """ % \ + basket_html = """ > %s""" % \ (CFG_SITE_URL, 'category=' + category + '&ln=' + ln + \ '#bsk' + str(bskid), diff --git a/modules/webbasket/lib/webbasket_dblayer.py b/modules/webbasket/lib/webbasket_dblayer.py index d7fc581a67..546b038158 100644 --- a/modules/webbasket/lib/webbasket_dblayer.py +++ b/modules/webbasket/lib/webbasket_dblayer.py @@ -37,93 +37,6 @@ from invenio.websession_config import CFG_WEBSESSION_USERGROUP_STATUS from invenio.search_engine import get_fieldvalues -########################### Table of contents ################################ -# -# NB. functions preceeded by a star use usergroup table -# -# 1. General functions -# - count_baskets -# - check_user_owns_basket -# - get_max_user_rights_on_basket -# -# 2. Personal baskets -# - get_personal_baskets_info_for_topic -# - get_all_personal_basket_ids_and_names_by_topic -# - get_all_personal_baskets_names -# - get_basket_name -# - is_personal_basket_valid -# - is_topic_valid -# - get_basket_topic -# - get_personal_topics_infos -# - rename_basket -# - rename_topic -# - move_baskets_to_topic -# - delete_basket -# - create_basket -# -# 3. Actions on baskets -# - get_basket_record -# - get_basket_content -# - get_basket_item -# - get_basket_item_title_and_URL -# - share_basket_with_group -# - update_rights -# - move_item -# - delete_item -# - add_to_basket -# - get_external_records_by_collection -# - store_external_records -# - store_external_urls -# - store_external_source -# - get_external_colid_and_url -# -# 4. Group baskets -# - get_group_basket_infos -# - get_group_name -# - get_all_group_basket_ids_and_names_by_group -# - (*) get_all_group_baskets_names -# - is_shared_to -# -# 5. External baskets (baskets user has subscribed to) -# - get_external_baskets_infos -# - get_external_basket_info -# - get_all_external_basket_ids_and_names -# - count_external_baskets -# - get_all_external_baskets_names -# -# 6. Public baskets (interface to subscribe to baskets) -# - get_public_basket_infos -# - get_public_basket_info -# - get_basket_general_infos -# - get_basket_owner_id -# - count_public_baskets -# - get_public_baskets_list -# - is_basket_public -# - subscribe -# - unsubscribe -# - is_user_subscribed_to_basket -# - count_subscribers -# - (*) get_groups_subscribing_to_basket -# - get_rights_on_public_basket -# -# 7. Annotating -# - get_notes -# - get_note -# - save_note -# - delete_note -# - note_belongs_to_item_in_basket_p -# -# 8. Usergroup functions -# - (*) get_group_infos -# - count_groups_user_member_of -# - (*) get_groups_user_member_of -# -# 9. auxilliary functions -# - __wash_sql_count -# - __decompress_last -# - create_pseudo_record -# - prettify_url - ########################## General functions ################################## def count_baskets(uid): @@ -433,7 +346,7 @@ def create_basket(uid, basket_name, topic): def get_all_items_in_user_personal_baskets(uid, topic="", - format='hb'): + of='hb'): """For the specified user, return all the items in their personal baskets, grouped by basket if local or as a list if external. If topic is set, return only that topic's items.""" @@ -441,11 +354,11 @@ def get_all_items_in_user_personal_baskets(uid, if topic: topic_clause = """AND ubsk.topic=%s""" params_local = (uid, uid, topic) - params_external = (uid, uid, topic, format) + params_external = (uid, uid, topic, of) else: topic_clause = "" params_local = (uid, uid) - params_external = (uid, uid, format) + params_external = (uid, uid, of) query_local = """ SELECT rec.id_bskBASKET, @@ -541,53 +454,7 @@ def get_all_user_topics(uid): ########################## Actions on baskets ################################# -def get_basket_record(bskid, recid, format='hb'): - """get record recid in basket bskid - """ - if recid < 0: - rec_table = 'bskEXTREC' - format_table = 'bskEXTFMT' - id_field = 'id_bskEXTREC' - sign = '-' - else: - rec_table = 'bibrec' - format_table = 'bibfmt' - id_field = 'id_bibrec' - sign = '' - query = """ - SELECT DATE_FORMAT(record.creation_date, '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s'), - DATE_FORMAT(record.modification_date, '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s'), - DATE_FORMAT(bskREC.date_added, '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s'), - user.nickname, - count(cmt.id_bibrec_or_bskEXTREC), - DATE_FORMAT(max(cmt.date_creation), '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s'), - fmt.value - - FROM bskREC LEFT JOIN user - ON bskREC.id_user_who_added_item=user.id - LEFT JOIN bskRECORDCOMMENT cmt - ON bskREC.id_bibrec_or_bskEXTREC=cmt.id_bibrec_or_bskEXTREC - LEFT JOIN %(rec_table)s record - ON (%(sign)sbskREC.id_bibrec_or_bskEXTREC=record.id) - LEFT JOIN %(format_table)s fmt - ON (record.id=fmt.%(id_field)s) - - WHERE bskREC.id_bskBASKET=%%s AND - bskREC.id_bibrec_or_bskEXTREC=%%s AND - fmt.format=%%s - - GROUP BY bskREC.id_bibrec_or_bskEXTREC - """ % {'rec_table': rec_table, - 'sign': sign, - 'format_table': format_table, - 'id_field':id_field} - params = (int(bskid), int(recid), format) - res = run_sql(query, params) - if res: - return __decompress_last(res[0]) - return () - -def get_basket_content(bskid, format='hb'): +def get_basket_content(bskid, of='hb'): """Get all records for a given basket.""" query = """ SELECT rec.id_bibrec_or_bskEXTREC, @@ -621,7 +488,7 @@ def get_basket_content(bskid, format='hb'): ORDER BY rec.score""" - params = (format, format, int(bskid)) + params = (of, of, int(bskid)) res = run_sql(query, params) @@ -631,7 +498,7 @@ def get_basket_content(bskid, format='hb'): return res return () -def get_basket_item(bskid, recid, format='hb'): +def get_basket_item(bskid, recid, of='hb'): """Get item (recid) for a given basket.""" query = """ SELECT rec.id_bibrec_or_bskEXTREC, @@ -660,7 +527,7 @@ def get_basket_item(bskid, recid, format='hb'): AND rec.id_bibrec_or_bskEXTREC=%s GROUP BY rec.id_bibrec_or_bskEXTREC ORDER BY rec.score""" - params = (format, format, bskid, recid) + params = (of, of, bskid, recid) res = run_sql(query, params) if res: queryU = """UPDATE bskBASKET SET nb_views=nb_views+1 WHERE id=%s""" @@ -1348,7 +1215,7 @@ def get_basket_share_level(bskid): def get_all_items_in_user_group_baskets(uid, group=0, - format='hb'): + of='hb'): """For the specified user, return all the items in their group baskets, grouped by basket if local or as a list if external. If group is set, return only that group's items.""" @@ -1356,11 +1223,11 @@ def get_all_items_in_user_group_baskets(uid, if group: group_clause = """AND ugbsk.id_usergroup=%s""" params_local = (group, uid) - params_external = (group, uid, format) + params_external = (group, uid, of) else: group_clause = "" params_local = (uid,) - params_external = (uid, format) + params_external = (uid, of) query_local = """ SELECT rec.id_bskBASKET, @@ -1631,7 +1498,7 @@ def get_all_external_baskets_names(uid, return run_sql(query, params) def get_all_items_in_user_public_baskets(uid, - format='hb'): + of='hb'): """For the specified user, return all the items in the public baskets they are subscribed to, grouped by basket if local or as a list if external.""" @@ -1679,7 +1546,7 @@ def get_all_items_in_user_public_baskets(uid, WHERE rec.id_bibrec_or_bskEXTREC < 0 ORDER BY rec.id_bskBASKET""" - params_external = (uid, uid, format) + params_external = (uid, uid, of) res_external = run_sql(query_external, params_external) @@ -1720,7 +1587,7 @@ def get_all_items_in_user_public_baskets_by_matching_notes(uid, return res -def get_all_items_in_all_public_baskets(format='hb'): +def get_all_items_in_all_public_baskets(of='hb'): """Return all the items in all the public baskets, grouped by basket if local or as a list if external.""" @@ -1758,7 +1625,7 @@ def get_all_items_in_all_public_baskets(format='hb'): WHERE rec.id_bibrec_or_bskEXTREC < 0 ORDER BY rec.id_bskBASKET""" - params_external = (format,) + params_external = (of,) res_external = run_sql(query_external, params_external) diff --git a/modules/webbasket/lib/webbasket_templates.py b/modules/webbasket/lib/webbasket_templates.py index 3d82ad8b5f..6e108bd9e1 100644 --- a/modules/webbasket/lib/webbasket_templates.py +++ b/modules/webbasket/lib/webbasket_templates.py @@ -41,9 +41,7 @@ CFG_SITE_RECORD from invenio.webuser import get_user_info from invenio.dateutils import convert_datetext_to_dategui -from invenio.webbasket_dblayer import get_basket_item_title_and_URL, \ - get_basket_ids_and_names -from invenio.bibformat import format_record +from invenio.webbasket_dblayer import get_basket_ids_and_names class Template: """Templating class for webbasket module""" @@ -3026,7 +3024,7 @@ def tmpl_basket_single_item_content(self, """ % _("The item you have selected does not exist.") else: - (recid, colid, dummy, last_cmt, val, dummy) = item + (recid, colid, dummy, dummy, val, dummy) = item if recid < 0: external_item_img = '%s ' % \ @@ -3889,8 +3887,7 @@ def tmpl_public_basket_single_item_content(self, """ % {'count': index_item, 'icon': external_item_img, 'content': colid >= 0 and val or val and self.tmpl_create_pseudo_item(val) or _("This record does not seem to exist any more"), - 'notes': notes, - 'ln': ln} + 'notes': notes,} item_html += """ """ @@ -4169,9 +4166,9 @@ def tmpl_create_export_as_list(self, recid) export_as_html = "" - for format in list_of_export_as_formats: + for of in list_of_export_as_formats: export_as_html += """%s, """ % \ - (href, format[1], format[0]) + (href, of[1], of[0]) if export_as_html: export_as_html = export_as_html[:-2] out = """ @@ -4361,13 +4358,13 @@ def create_add_box_select_options(category, if len(personal_basket_list) == 1: bskids = personal_basket_list[0][1].split(',') if len(bskids) == 1: - b = CFG_WEBBASKET_CATEGORIES['PRIVATE'] + '_' + bskids[0] + b = CFG_WEBBASKET_CATEGORIES['PRIVATE'] + '_' + bskids[0] elif len(group_basket_list) == 1: bskids = group_basket_list[0][1].split(',') if len(bskids) == 1: - b = CFG_WEBBASKET_CATEGORIES['GROUP'] + '_' + bskids[0] + b = CFG_WEBBASKET_CATEGORIES['GROUP'] + '_' + bskids[0] - # Create the s and """ % ('* ' + _('Your personal baskets') + ' *',) @@ -4390,7 +4387,7 @@ def create_add_box_select_options(category, out += """ """ - # Create the s and """ % ('* ' + _('Your group baskets') + ' *',) diff --git a/modules/webbasket/lib/webbasket_webinterface.py b/modules/webbasket/lib/webbasket_webinterface.py index f47c6cf01c..13e5d37f60 100644 --- a/modules/webbasket/lib/webbasket_webinterface.py +++ b/modules/webbasket/lib/webbasket_webinterface.py @@ -409,7 +409,7 @@ def search(self, req, form): p=argd['p'], b=argd['b'], n=argd['n'], -# format=argd['of'], +# record_format=argd['of'], ln=argd['ln']) # register event in webstat diff --git a/modules/webcomment/lib/webcomment.py b/modules/webcomment/lib/webcomment.py index 8a40013333..752c276dd2 100644 --- a/modules/webcomment/lib/webcomment.py +++ b/modules/webcomment/lib/webcomment.py @@ -410,21 +410,22 @@ def perform_request_report(cmt_id, client_ip_address, uid=-1): params = (cmt_id, uid, client_ip_address, action_date, action_code) run_sql(query, params) if nb_abuse_reports % CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN == 0: - (cmt_id2, + (dummy, id_bibrec, id_user, cmt_body, cmt_date, cmt_star, - cmt_vote, cmt_nb_votes_total, + dummy, + dummy, cmt_title, cmt_reported, - round_name, - restriction) = query_get_comment(cmt_id) + dummy, + dummy) = query_get_comment(cmt_id) (user_nb_abuse_reports, user_votes, user_nb_votes_total) = query_get_user_reports_and_votes(int(id_user)) - (nickname, user_email, last_login) = query_get_user_contact_info(id_user) + (nickname, user_email, dummy) = query_get_user_contact_info(id_user) from_addr = '%s Alert Engine <%s>' % (CFG_SITE_NAME, CFG_WEBALERT_ALERT_ENGINE_EMAIL) comment_collection = get_comment_collection(cmt_id) to_addrs = get_collection_moderators(comment_collection) @@ -449,12 +450,10 @@ def perform_request_report(cmt_id, client_ip_address, uid=-1): ---end body--- Please go to the record page %(comment_admin_link)s to delete this message if necessary. A warning will be sent to the user in question.''' % \ - { 'cfg-report_max' : CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN, - 'nickname' : nickname, + { 'nickname' : nickname, 'user_email' : user_email, 'uid' : id_user, 'user_nb_abuse_reports' : user_nb_abuse_reports, - 'user_votes' : user_votes, 'votes' : CFG_WEBCOMMENT_ALLOW_REVIEWS and \ "total number of positive votes\t= %s\n\t\ttotal number of negative votes\t= %s" % \ (user_votes, (user_nb_votes_total - user_votes)) or "\n", @@ -815,7 +814,6 @@ def query_add_comment_or_remark(reviews=0, recID=0, uid=-1, msg="", #change general unicode back to utf-8 msg = msg.encode('utf-8') note = note.encode('utf-8') - msg_original = msg (restriction, round_name) = get_record_status(recID) if attached_files is None: attached_files = {} @@ -1057,7 +1055,7 @@ def get_users_subscribed_to_discussion(recID, check_authorizations=True): uid = row[0] if check_authorizations: user_info = collect_user_info(uid) - (auth_code, auth_msg) = check_user_can_view_comments(user_info, recID) + (auth_code, dummy) = check_user_can_view_comments(user_info, recID) else: # Don't check and grant access auth_code = False @@ -1485,7 +1483,6 @@ def perform_request_add_comment_or_remark(recID=0, if warnings is None: warnings = [] - actions = ['DISPLAY', 'REPLY', 'SUBMIT'] _ = gettext_set_language(ln) ## check arguments @@ -1635,18 +1632,18 @@ def notify_admin_of_new_comment(comID): id_user, body, date_creation, - star_score, nb_votes_yes, nb_votes_total, + star_score, dummy, dummy, title, - nb_abuse_reports, round_name, restriction) = comment + dummy, dummy, dummy) = comment else: return user_info = query_get_user_contact_info(id_user) if len(user_info) > 0: - (nickname, email, last_login) = user_info + (nickname, email, dummy) = user_info if not len(nickname) > 0: nickname = email.split('@')[0] else: - nickname = email = last_login = "ERROR: Could not retrieve" + nickname = email = "ERROR: Could not retrieve" review_stuff = ''' Star score = %s diff --git a/modules/webmessage/lib/webmessage.py b/modules/webmessage/lib/webmessage.py index 1e5cd9f2f6..4462463e52 100644 --- a/modules/webmessage/lib/webmessage.py +++ b/modules/webmessage/lib/webmessage.py @@ -191,7 +191,7 @@ def perform_request_write(uid, @type ln: string @return: body with warnings. """ - warnings = [] + body = "" _ = gettext_set_language(ln) msg_from_nickname = "" diff --git a/modules/websearch/lib/websearch_yoursearches.py b/modules/websearch/lib/websearch_yoursearches.py index 25c621d0e7..c2d4bd8161 100644 --- a/modules/websearch/lib/websearch_yoursearches.py +++ b/modules/websearch/lib/websearch_yoursearches.py @@ -19,10 +19,9 @@ __revision__ = "$Id$" -from invenio.config import CFG_SITE_LANG, CFG_SITE_SECURE_URL +from invenio.config import CFG_SITE_LANG from invenio.dbquery import run_sql from invenio.messages import gettext_set_language -from invenio.webuser import isGuestUser from urllib import quote from invenio.webalert import count_user_alerts_for_given_query diff --git a/modules/websession/lib/webaccount.py b/modules/websession/lib/webaccount.py index bc4fca0403..2f8d373428 100644 --- a/modules/websession/lib/webaccount.py +++ b/modules/websession/lib/webaccount.py @@ -29,10 +29,8 @@ CFG_SITE_LANG, \ CFG_SITE_SUPPORT_EMAIL, \ CFG_SITE_ADMIN_EMAIL, \ - CFG_SITE_SECURE_URL, \ CFG_VERSION, \ CFG_SITE_RECORD -from invenio.access_control_engine import acc_authorize_action from invenio.access_control_config import CFG_EXTERNAL_AUTHENTICATION, \ SUPERADMINROLE, CFG_EXTERNAL_AUTH_DEFAULT from invenio.dbquery import run_sql @@ -65,9 +63,7 @@ def perform_info(req, ln): return websession_templates.tmpl_account_info( ln = ln, uid = uid, - guest = int(user_info['guest']), - CFG_CERN_SITE = CFG_CERN_SITE, - ) + guest = int(user_info['guest'])) def perform_display_external_user_settings(settings, ln): """show external user settings which is a dictionary.""" @@ -235,7 +231,7 @@ def superuser_account_warnings(): # no account nick-named `admin' exists; keep on going res1 = [] - for user in res1: + for dummy in res1: warning_array.append("warning_empty_admin_password") #Check if the admin email has been changed from the default diff --git a/modules/websession/lib/webgroup.py b/modules/websession/lib/webgroup.py index 99a2e780b6..ff638bd22b 100644 --- a/modules/websession/lib/webgroup.py +++ b/modules/websession/lib/webgroup.py @@ -781,7 +781,6 @@ def account_user_groups(uid, ln=CFG_SITE_LANG): out = websession_templates.tmpl_account_user_groups( nb_admin_groups, - nb_member_groups, nb_total_groups, ln = ln) diff --git a/modules/websession/lib/websession_templates.py b/modules/websession/lib/websession_templates.py index cf09bb43f2..649229ec1c 100644 --- a/modules/websession/lib/websession_templates.py +++ b/modules/websession/lib/websession_templates.py @@ -69,7 +69,6 @@ def tmpl_back_form(self, ln, message, url, link): 'message' : message, 'url' : url, 'link' : link, - 'ln' : ln } return out @@ -315,7 +314,6 @@ def tmpl_user_preferences(self, ln, email, email_disabled, password_disabled, ni """ % { 'change_pass' : _("If you want to change your password, please enter the old one and set the new value in the form below."), - 'mandatory' : _("mandatory"), 'old_password' : _("Old password"), 'new_password' : _("New password"), 'csrf_token': cgi.escape(csrf_token, True), @@ -562,7 +560,7 @@ def tmpl_lost_password_form(self, ln): return out - def tmpl_account_info(self, ln, uid, guest, CFG_CERN_SITE): + def tmpl_account_info(self, ln, uid, guest): """ Displays the account information @@ -573,8 +571,6 @@ def tmpl_account_info(self, ln, uid, guest, CFG_CERN_SITE): - 'uid' *string* - The user id - 'guest' *boolean* - If the user is guest - - - 'CFG_CERN_SITE' *boolean* - If the site is a CERN site """ # load the right message language @@ -645,7 +641,7 @@ def tmpl_account_info(self, ln, uid, guest, CFG_CERN_SITE): return out - def tmpl_warning_guest_user(self, ln, type): + def tmpl_warning_guest_user(self, warning_type, ln): """ Displays a warning message about the specified type @@ -653,16 +649,16 @@ def tmpl_warning_guest_user(self, ln, type): - 'ln' *string* - The language to display the interface in - - 'type' *string* - The type of data that will get lost in case of guest account (for the moment: 'alerts' or 'baskets') + - 'warning_type' *string* - The type of data that will get lost in case of guest account (for the moment: 'alerts' or 'baskets') """ # load the right message language _ = gettext_set_language(ln) - if (type=='baskets'): + if (warning_type == 'baskets'): msg = _("You are logged in as a guest user, so your baskets will disappear at the end of the current session.") + ' ' - elif (type=='alerts'): + elif (warning_type == 'alerts'): msg = _("You are logged in as a guest user, so your alerts will disappear at the end of the current session.") + ' ' - elif (type=='searches'): + elif (warning_type == 'searches'): msg = _("You are logged in as a guest user, so your searches will disappear at the end of the current session.") + ' ' msg += _("If you wish you can %(x_url_open)slogin or register here%(x_url_close)s.") % {'x_url_open': '', 'x_url_close': ''} @@ -1822,7 +1818,7 @@ def tmpl_display_member_groups(self, groups, ln=CFG_SITE_LANG): """ %(_("You are not a member of any groups."),) for group_data in groups: - (id, name, description) = group_data + (dummy, name, description) = group_data group_text += """ %s @@ -1885,7 +1881,7 @@ def tmpl_display_external_groups(self, groups, ln=CFG_SITE_LANG): """ %(_("You are not a member of any external groups."),) for group_data in groups: - (id, name, description) = group_data + (dummy, name, description) = group_data group_text += """ %s @@ -2645,13 +2641,11 @@ def tmpl_delete_msg(self, def tmpl_account_user_groups( self, nb_admin_groups = 0, - nb_member_groups = 0, nb_total_groups = 0, ln = CFG_SITE_LANG): """ Information on the user's groups for the "Your Account" page @param nb_admin_groups: number of groups the user is admin of - @param nb_member_groups: number of groups the user is member of @param nb_total_groups: number of groups the user belongs to @param ln: language return: html output. diff --git a/modules/websession/lib/websession_webinterface.py b/modules/websession/lib/websession_webinterface.py index 73599cb9c1..ca02481616 100644 --- a/modules/websession/lib/websession_webinterface.py +++ b/modules/websession/lib/websession_webinterface.py @@ -26,7 +26,6 @@ import cgi from datetime import timedelta -import os import re from invenio.config import \ @@ -38,7 +37,6 @@ CFG_SITE_SUPPORT_EMAIL, \ CFG_SITE_SECURE_URL, \ CFG_SITE_URL, \ - CFG_CERN_SITE, \ CFG_WEBSESSION_RESET_PASSWORD_EXPIRE_IN_DAYS, \ CFG_OPENAIRE_SITE from invenio import webuser @@ -61,7 +59,7 @@ InvenioWebAccessMailCookieDeletedError, mail_cookie_check_authorize_action from invenio.access_control_config import CFG_WEBACCESS_WARNING_MSGS, \ CFG_EXTERNAL_AUTH_USING_SSO, CFG_EXTERNAL_AUTH_LOGOUT_SSO, \ - CFG_EXTERNAL_AUTHENTICATION, CFG_EXTERNAL_AUTH_SSO_REFRESH, \ + CFG_EXTERNAL_AUTHENTICATION, \ CFG_OPENID_CONFIGURATIONS, CFG_OAUTH2_CONFIGURATIONS, \ CFG_OAUTH1_CONFIGURATIONS, CFG_OAUTH2_PROVIDERS, CFG_OAUTH1_PROVIDERS, \ CFG_OPENID_PROVIDERS, CFG_OPENID_AUTHENTICATION, \ @@ -130,7 +128,7 @@ def access(self, req, form): body += "

    " + _("You can now go to %(x_url_open)syour account page%(x_url_close)s.") % {'x_url_open' : '' % args['ln'], 'x_url_close' : ''} + "

    " return page(title=_("Email address successfully activated"), body=body, req=req, language=args['ln'], uid=webuser.getUid(req), lastupdated=__lastupdated__, navmenuid='youraccount', secure_page_p=1) - except InvenioWebAccessMailCookieDeletedError, e: + except InvenioWebAccessMailCookieDeletedError: body = "

    " + _("You have already confirmed the validity of your email address!") + "

    " if CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS == 1: body += "

    " + _("Please, wait for the administrator to " @@ -910,7 +908,7 @@ def login(self, req, form): if args['action']: cookie = args['action'] try: - action, arguments = mail_cookie_check_authorize_action(cookie) + dummy, dummy = mail_cookie_check_authorize_action(cookie) except InvenioWebAccessMailCookieError: pass if not CFG_EXTERNAL_AUTH_USING_SSO: From 99bbe888b52a778ff03f2a42f512943f511f1c15 Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Thu, 27 Mar 2014 11:24:20 +0100 Subject: [PATCH 34/59] Miscutil: fix get_authenticated_mechanize_browser * Fixes the string checked in get_authenticated_mechanize_browser to make sure that the user was logged in correctly. --- modules/miscutil/lib/testutils.py | 2 +- modules/websession/lib/websession_templates.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/modules/miscutil/lib/testutils.py b/modules/miscutil/lib/testutils.py index f1773104c2..2c3ea644d6 100644 --- a/modules/miscutil/lib/testutils.py +++ b/modules/miscutil/lib/testutils.py @@ -223,7 +223,7 @@ def get_authenticated_mechanize_browser(username="guest", password=""): browser.submit() username_account_page_body = browser.response().read() try: - username_account_page_body.index("You are logged in as %s." % username) + username_account_page_body.index("You are logged in as %s." % ("" + cgi.escape(username) + ""),) except ValueError: raise InvenioTestUtilsBrowserException('ERROR: Cannot login as %s.' % username) return browser diff --git a/modules/websession/lib/websession_templates.py b/modules/websession/lib/websession_templates.py index 649229ec1c..f05030ade1 100644 --- a/modules/websession/lib/websession_templates.py +++ b/modules/websession/lib/websession_templates.py @@ -808,13 +808,17 @@ def tmpl_account_page( # While there are still items to display, get the first item, # removing it from the list of items to display. item = items.pop(0) - # The following "1-liner" does the following (>= Python 2.5): + # The following "one-liner" does the following (>= Python 2.5): # * For each of the 3 lists (1 for each column) # calculate the sum of the length of its items (see the lambda). - # The lenght of an itme is the literal length of the string - # to be displayed, which includes HTML code. This of course is - # not the most accurate way, but a good approximation and also + # The lenght of an item is the literal length of the string + # to be displayed, which includes HTML code. This, of course, is + # not the most accurate way, but it is a good approximation and also # very efficient. + # [A more accurate way would be to wash the HTML tags first + # (HTMLParser/re/...) and then compare the string sizes. But we should + # still account for line breaks, paragraphs, list items and other HTML + # tags that introduce white space.] # * Pick the list that has the smallest sum. # * Append the current item to that list. min(items_in_column_1_of_3, From a28b8ddc7ef575ca114c0eb988d687f080dfeb4f Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Thu, 27 Mar 2014 19:21:12 +0100 Subject: [PATCH 35/59] Websearch: remove mentions of total searches * Removes mentions of the number of total searches the user has performed in favor of only the unique searches to avoid possible confusion. --- modules/websearch/lib/websearch_templates.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/modules/websearch/lib/websearch_templates.py b/modules/websearch/lib/websearch_templates.py index 1c82a761ab..467a40db58 100644 --- a/modules/websearch/lib/websearch_templates.py +++ b/modules/websearch/lib/websearch_templates.py @@ -5148,14 +5148,12 @@ def tmpl_yoursearches_display(self, # Diplay a message about the number of searches. if p: - msg = _("You have performed %(searches_distinct)s unique searches in a total of %(searches_total)s searches including the term %(p)s.") % \ + msg = _("You have performed %(searches_distinct)s unique searches including the term %(p)s.") % \ {'searches_distinct': '' + str(nb_queries_distinct) + '', - 'searches_total': '' + str(nb_queries_total) + '', 'p': '' + cgi.escape(p) + ''} else: - msg = _("You have performed %(searches_distinct)s unique searches in a total of %(searches_total)s searches.") % \ - {'searches_distinct': '' + str(nb_queries_distinct) + '', - 'searches_total': '' + str(nb_queries_total) + ''} + msg = _("You have performed %(searches_distinct)s unique searches.") % \ + {'searches_distinct': '' + str(nb_queries_distinct) + '',} out = '

    ' + msg + '

    ' # Search form @@ -5263,11 +5261,10 @@ def tmpl_account_user_searches(self, unique, total, ln = CFG_SITE_LANG): _ = gettext_set_language(ln) if unique > 0: - out = _("You have performed %(x_url_open)s%(unique)s unique searches%(x_url_close)s in a total of %(total)s searches.") % \ + out = _("You have performed %(x_url_open)s%(x_unique)s unique searches%(x_url_close)s.") % \ {'x_url_open' : '' % (CFG_SITE_SECURE_URL, ln), - 'unique' : str(unique), - 'x_url_close' : '', - 'total' : str(total),} + 'x_unique' : str(unique), + 'x_url_close' : '',} else: out = _("You have not searched for anything yet. You may want to start by the %(x_url_open)ssearch interface%(x_url_close)s first.") % \ {'x_url_open' : '' % (CFG_SITE_SECURE_URL, ln), From c0eef8481a9520ca453d87da21b4f057fedbe580 Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Fri, 28 Mar 2014 11:21:42 +0100 Subject: [PATCH 36/59] General: fix failing tests --- modules/webhelp/web/hacking/test-suite.webdoc | 2 +- modules/websession/lib/webgroup_regression_tests.py | 4 ++-- modules/websession/lib/webuser_regression_tests.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/webhelp/web/hacking/test-suite.webdoc b/modules/webhelp/web/hacking/test-suite.webdoc index ec94fac788..282d8174ac 100644 --- a/modules/webhelp/web/hacking/test-suite.webdoc +++ b/modules/webhelp/web/hacking/test-suite.webdoc @@ -382,7 +382,7 @@ browser.submit() username_account_page_body = browser.response().read() try: string.index(username_account_page_body, - "You are logged in as userfoo.") + "You are logged in as <strong>userfoo</strong>.") except ValueError: self.fail('ERROR: Cannot login as userfoo.') diff --git a/modules/websession/lib/webgroup_regression_tests.py b/modules/websession/lib/webgroup_regression_tests.py index 5bb431bf92..38360b1924 100644 --- a/modules/websession/lib/webgroup_regression_tests.py +++ b/modules/websession/lib/webgroup_regression_tests.py @@ -116,7 +116,7 @@ def test_external_groups_visibility_groupspage(self): browser['p_pw'] = '' browser.submit() - expected_response = "You are logged in as admin" + expected_response = "You are logged in as admin." login_response_body = browser.response().read() try: login_response_body.index(expected_response) @@ -151,7 +151,7 @@ def test_external_groups_visibility_messagespage(self): browser['p_pw'] = '' browser.submit() - expected_response = "You are logged in as admin" + expected_response = "You are logged in as admin." login_response_body = browser.response().read() try: login_response_body.index(expected_response) diff --git a/modules/websession/lib/webuser_regression_tests.py b/modules/websession/lib/webuser_regression_tests.py index 03d8189522..6bc70c7ebb 100644 --- a/modules/websession/lib/webuser_regression_tests.py +++ b/modules/websession/lib/webuser_regression_tests.py @@ -63,7 +63,7 @@ def test_password_setting(self): browser['p_pw'] = '' browser.submit() - expected_response = "You are logged in as admin" + expected_response = "You are logged in as admin." login_response_body = browser.response().read() try: login_response_body.index(expected_response) @@ -196,7 +196,7 @@ def test_select_records_per_group(self): browser['p_pw'] = '' browser.submit() - expected_response = "You are logged in as admin" + expected_response = "You are logged in as admin." login_response_body = browser.response().read() try: login_response_body.index(expected_response) @@ -261,7 +261,7 @@ def test_select_records_per_group(self): browser['p_pw'] = '' browser.submit() - expected_response = "You are logged in as admin" + expected_response = "You are logged in as admin." login_response_body = browser.response().read() try: login_response_body.index(expected_response) From 358fe01743fa868ff5a0cc3eb8b3bee98ac5edf4 Mon Sep 17 00:00:00 2001 From: "Esteban J. G. Gabancho" Date: Fri, 17 Jul 2015 16:59:40 +0200 Subject: [PATCH 37/59] WebStyle: wsgi handler IP parser enhancement * BETTER Cleans the port from the user's IP preventing issues in future manipulations of it. Reviewed-by: Samuele Kaplun Signed-off-by: Esteban J. G. Gabancho --- modules/webstyle/lib/webinterface_handler_wsgi.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/modules/webstyle/lib/webinterface_handler_wsgi.py b/modules/webstyle/lib/webinterface_handler_wsgi.py index 580d595375..45f1bea933 100644 --- a/modules/webstyle/lib/webinterface_handler_wsgi.py +++ b/modules/webstyle/lib/webinterface_handler_wsgi.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # This file is part of Invenio. -# Copyright (C) 2009, 2010, 2011, 2012 CERN. +# Copyright (C) 2009, 2010, 2011, 2012, 2015 CERN. # # Invenio is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -62,10 +62,9 @@ # any src usage of an external website _RE_HTTPS_REPLACES = re.compile(r"\b((?:src\s*=|url\s*\()\s*[\"']?)http\://", re.I) -# Regexp to verify that the IP starts with a number (filter cases where 'unknown') -# It is faster to verify only the start (585 ns) compared with verifying -# the whole ip address - re.compile('^\d+\.\d+\.\d+\.\d+$') (1.01 µs) -_RE_IPADDRESS_START = re.compile(r"^\d+\.") +# Regexp to verify the IP (filter cases where 'unknown'). +_RE_IPADDRESS = re.compile(r"^\d+(\.\d+){3}$") +_RE_IPADDRESS_WITH_PORT = re.compile(r"^\d+(\.\d+){3}:\d+$") # Regexp to match IE User-Agent _RE_BAD_MSIE = re.compile(r"MSIE\s+(\d+\.\d+)") @@ -277,8 +276,11 @@ def get_remote_ip(self): # we trust this proxy ip_list = self.__headers_in['X-FORWARDED-FOR'].split(',') for ip in ip_list: - if _RE_IPADDRESS_START.match(ip): + if _RE_IPADDRESS.match(ip): return ip + # Probably because behind a proxy + elif _RE_IPADDRESS_WITH_PORT.match(ip): + return ip[:ip.index(':')] # no IP has the correct format, return a default IP return '10.0.0.10' else: From 40fc4198542d841f0ada1534bbc10d622c59833b Mon Sep 17 00:00:00 2001 From: "Esteban J. G. Gabancho" Date: Fri, 17 Jul 2015 17:01:05 +0200 Subject: [PATCH 38/59] WebStyle: kwalitee fix * Improves `webinterface_handler_wsgi` PEP8 code style. Signed-off-by: Esteban J. G. Gabancho --- .../webstyle/lib/webinterface_handler_wsgi.py | 218 ++++++++++-------- 1 file changed, 118 insertions(+), 100 deletions(-) diff --git a/modules/webstyle/lib/webinterface_handler_wsgi.py b/modules/webstyle/lib/webinterface_handler_wsgi.py index 45f1bea933..cc43dbfb38 100644 --- a/modules/webstyle/lib/webinterface_handler_wsgi.py +++ b/modules/webstyle/lib/webinterface_handler_wsgi.py @@ -16,40 +16,50 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. -"""mod_python->WSGI Framework""" +"""mod_python->WSGI Framework.""" -import sys -import os -import re import cgi import gc import inspect +import os +import re import socket import logging + from fnmatch import fnmatch from urlparse import urlparse, urlunparse - -from wsgiref.validate import validator from wsgiref.util import FileWrapper +from wsgiref.validate import validator if __name__ != "__main__": # Chances are that we are inside mod_wsgi. - ## You can't write to stdout in mod_wsgi, but some of our - ## dependecies do this! (e.g. 4Suite) + # You can't write to stdout in mod_wsgi, but some of our + # dependecies do this! (e.g. 4Suite) sys.stdout = sys.stderr -from invenio.urlutils import redirect_to_url +from invenio.config import ( + CFG_DEVEL_SITE, + CFG_SITE_LANG, + CFG_SITE_SECURE_URL, + CFG_SITE_URL, + CFG_WEBDIR, + CFG_WEBSTYLE_HTTP_STATUS_ALERT_LIST, + CFG_WEBSTYLE_REVERSE_PROXY_IPS +) +from invenio.errorlib import get_pretty_traceback, register_exception from invenio.session import get_session -from invenio.webinterface_handler import CFG_HAS_HTTPS_SUPPORT, CFG_FULL_HTTPS +from invenio.urlutils import redirect_to_url +from invenio.webinterface_handler import CFG_FULL_HTTPS, CFG_HAS_HTTPS_SUPPORT +from invenio.webinterface_handler_config import ( + DONE, + HTTP_INTERNAL_SERVER_ERROR, + HTTP_NOT_FOUND, + HTTP_STATUS_MAP, + OK, + SERVER_RETURN +) +from invenio.webinterface_handler_wsgi_utils import FieldStorage, table from invenio.webinterface_layout import invenio_handler -from invenio.webinterface_handler_wsgi_utils import table, FieldStorage -from invenio.webinterface_handler_config import \ - HTTP_STATUS_MAP, SERVER_RETURN, OK, DONE, \ - HTTP_NOT_FOUND, HTTP_INTERNAL_SERVER_ERROR -from invenio.config import CFG_WEBDIR, CFG_SITE_LANG, \ - CFG_WEBSTYLE_HTTP_STATUS_ALERT_LIST, CFG_DEVEL_SITE, CFG_SITE_URL, \ - CFG_SITE_SECURE_URL, CFG_WEBSTYLE_REVERSE_PROXY_IPS -from invenio.errorlib import register_exception, get_pretty_traceback # Static files are usually handled directly by the webserver (e.g. Apache) # However in case WSGI is required to handle static files too (such @@ -71,31 +81,41 @@ def _http_replace_func(match): - ## src external_site -> CFG_SITE_SECURE_URL/sslredirect/external_site + # src external_site -> CFG_SITE_SECURE_URL/sslredirect/external_site return match.group(1) + CFG_SITE_SECURE_URL + '/sslredirect/' _ESCAPED_CFG_SITE_URL = cgi.escape(CFG_SITE_URL, True) _ESCAPED_CFG_SITE_SECURE_URL = cgi.escape(CFG_SITE_SECURE_URL, True) + + def https_replace(html): html = html.replace(_ESCAPED_CFG_SITE_URL, _ESCAPED_CFG_SITE_SECURE_URL) return _RE_HTTPS_REPLACES.sub(_http_replace_func, html) + class InputProcessed(object): + """ Auxiliary class used when reading input. + @see: . """ + def read(self, *args): raise EOFError('The wsgi.input stream has already been consumed') readline = readlines = __iter__ = read + class SimulatedModPythonRequest(object): + """ mod_python like request object. + Minimum and cleaned implementation to make moving out of mod_python easy. @see: """ + def __init__(self, environ, start_response): self.__environ = environ self.__start_response = start_response @@ -111,7 +131,7 @@ def __init__(self, environ, start_response): self.__allowed_methods = [] self.__cleanups = [] self.headers_out = self.__headers - ## See: + # See: self.__write = None self.__write_error = False self.__errors = environ['wsgi.errors'] @@ -122,7 +142,7 @@ def __init__(self, environ, start_response): self.track_writings = False self.__what_was_written = "" self.__cookies_out = {} - self.g = {} ## global dictionary in case it's needed + self.g = {} # global dictionary in case it's needed for key, value in environ.iteritems(): if key.startswith('HTTP_'): self.__headers_in[key[len('HTTP_'):].replace('_', '-')] = value @@ -132,14 +152,12 @@ def __init__(self, environ, start_response): self.__headers_in['content-type'] = environ['CONTENT_TYPE'] def set_cookie(self, cookie): - """ - This function is used to cumulate identical cookies. - """ + """This function is used to cumulate identical cookies.""" self.__cookies_out[cookie.name] = cookie def _write_cookies(self): if self.__cookies_out: - if not self.headers_out.has_key("Set-Cookie"): + if "Set-Cookie" not in self.headers_out: g = _RE_BAD_MSIE.search(self.headers_in.get('User-Agent', "MSIE 6.0")) bad_msie = g and float(g.group(1)) < 9.0 if not (bad_msie and self.is_https()): @@ -154,13 +172,13 @@ def get_post_form(self): self.__tainted = True post_form = self.__environ.get('wsgi.post_form') input = self.__environ['wsgi.input'] - if (post_form is not None - and post_form[0] is input): + if (post_form is not None and + post_form[0] is input): return post_form[2] # This must be done to avoid a bug in cgi.FieldStorage self.__environ.setdefault('QUERY_STRING', '') - ## Video handler hack: + # Video handler hack: uri = self.__environ['PATH_INFO'] if uri.endswith("upload_video"): tmp_shared = True @@ -211,11 +229,12 @@ def flush(self): self.__what_was_written += self.__buffer except IOError, err: if "failed to write data" in str(err) or "client connection closed" in str(err): - ## Let's just log this exception without alerting the admin: + # Let's just log this exception without alerting the admin: err.level=logging.INFO register_exception(req=self) - self.__write_error = True ## This flag is there just - ## to not report later other errors to the admin. + self.__write_error = True + # This flag is there just + # to not report later other errors to the admin. else: raise self.__buffer = '' @@ -236,7 +255,7 @@ def send_http_header(self): if self.__allowed_methods and self.__status.startswith('405 ') or self.__status.startswith('501 '): self.__headers['Allow'] = ', '.join(self.__allowed_methods) - ## See: + # See: #print self.__low_level_headers self.__write = self.__start_response(self.__status, self.__low_level_headers) self.__response_sent_p = True @@ -268,9 +287,9 @@ def get_args(self): def get_remote_ip(self): if 'X-FORWARDED-FOR' in self.__headers_in and \ - self.__headers_in.get('X-FORWARDED-SERVER', '') == \ - self.__headers_in.get('X-FORWARDED-HOST', '') == \ - urlparse(CFG_SITE_URL)[1]: + self.__headers_in.get('X-FORWARDED-SERVER', '') == \ + self.__headers_in.get('X-FORWARDED-HOST', '') == \ + urlparse(CFG_SITE_URL)[1]: # we are using proxy setup if self.__environ.get('REMOTE_ADDR') in CFG_WEBSTYLE_REVERSE_PROXY_IPS: # we trust this proxy @@ -285,11 +304,12 @@ def get_remote_ip(self): return '10.0.0.10' else: # we don't trust this proxy - register_exception(prefix="You are running in a proxy configuration, but the " + \ - "CFG_WEBSTYLE_REVERSE_PROXY_IPS variable does not contain " + \ - "the IP of your proxy, thus the remote IP addresses of your " + \ - "clients are not trusted. Please configure this variable.", - alert_admin=True) + register_exception( + prefix="You are running in a proxy configuration, but the " + "CFG_WEBSTYLE_REVERSE_PROXY_IPS variable does not contain " + "the IP of your proxy, thus the remote IP addresses of your " + "clients are not trusted. Please configure this variable.", + alert_admin=True) return '10.0.0.11' return self.__environ.get('REMOTE_ADDR') @@ -340,7 +360,7 @@ def sendfile(self, path, offset=0, the_len=-1): raise except IOError, err: if "failed to write data" in str(err) or "client connection closed" in str(err): - ## Let's just log this exception without alerting the admin: + # Let's just log this exception without alerting the admin: register_exception(req=self) else: raise @@ -394,10 +414,10 @@ def readline(self, hint=None): try: return self.__environ['wsgi.input'].readline(hint) except TypeError: - ## the hint param is not part of wsgi pep, although - ## it's great to exploit it in when reading FORM - ## with large files, in order to avoid filling up the memory - ## Too bad it's not there :-( + # the hint param is not part of wsgi pep, although + # it's great to exploit it in when reading FORM + # with large files, in order to avoid filling up the memory + # Too bad it's not there :-( return self.__environ['wsgi.input'].readline() def readlines(self, hint=None): @@ -430,9 +450,8 @@ def __str__(self): return out def get_original_wsgi_environment(self): - """ - Return the original WSGI environment used to initialize this request - object. + """Return the original WSGI environment used to initialize this object. + @return: environ, start_response @raise AssertionError: in case the environment has been altered, i.e. either the input has been consumed or something has already been @@ -468,9 +487,10 @@ def get_environ(self): referer = property(get_referer) what_was_written = property(get_what_was_written) + def alert_admin_for_server_status_p(status, referer): - """ - Check the configuration variable + """Check the configuration variable. + CFG_WEBSTYLE_HTTP_STATUS_ALERT_LIST to see if the exception should be registered and the admin should be alerted. """ @@ -479,20 +499,19 @@ def alert_admin_for_server_status_p(status, referer): pattern = pattern.lower() must_have_referer = False if pattern.endswith('r'): - ## e.g. "404 r" + # e.g. "404 r" must_have_referer = True - pattern = pattern[:-1].strip() ## -> "404" + pattern = pattern[:-1].strip() # -> "404" if fnmatch(status, pattern) and (not must_have_referer or referer): return True return False + def application(environ, start_response): - """ - Entry point for wsgi. - """ - ## Needed for mod_wsgi, see: + """Entry point for wsgi.""" + # Needed for mod_wsgi, see: req = SimulatedModPythonRequest(environ, start_response) - #print 'Starting mod_python simulation' + # print 'Starting mod_python simulation' try: try: if (CFG_FULL_HTTPS or (CFG_HAS_HTTPS_SUPPORT and get_session(req).need_https)) and not req.is_https(): @@ -505,7 +524,7 @@ def application(environ, start_response): # Compute the new path plain_path = original_parts[2] plain_path = secure_prefix_parts[2] + \ - plain_path[len(plain_prefix_parts[2]):] + plain_path[len(plain_prefix_parts[2]):] # ...and recompose the complete URL final_parts = list(secure_prefix_parts) @@ -533,8 +552,8 @@ def application(environ, start_response): if status not in (OK, DONE): req.status = status req.headers_out['content-type'] = 'text/html' - admin_to_be_alerted = alert_admin_for_server_status_p(status, - req.headers_in.get('referer')) + admin_to_be_alerted = alert_admin_for_server_status_p( + status, req.headers_in.get('referer')) if admin_to_be_alerted: register_exception(req=req, alert_admin=True) if not req.response_sent_p: @@ -557,12 +576,12 @@ def application(environ, start_response): return generate_error_page(req, page_already_started=True) finally: try: - ## Let's save the session. + # Let's save the session. session = get_session(req) try: if req.is_https() or not session.need_https: - ## We save the session only if it's safe to do it, i.e. - ## if we well had a valid session. + # We save the session only if it's safe to do it, i.e. + # if we well had a valid session. session.dirty = True session.save() if 'user_info' in req._session: @@ -570,35 +589,31 @@ def application(environ, start_response): finally: del session except Exception: - ## What could have gone wrong? + # What could have gone wrong? register_exception(req=req, alert_admin=True) if hasattr(req, '_session'): - ## The session handler saves for caching a request_wrapper - ## in req. - ## This saves req as an attribute, creating a circular - ## reference. - ## Since we have have reached the end of the request handler - ## we can safely drop the request_wrapper so to avoid - ## memory leaks. + # The session handler saves for caching a request_wrapper in req. + # This saves req as an attribute, creating a circular reference. + # Since we have have reached the end of the request handler + # we can safely drop the request_wrapper so to avoid memory leaks. delattr(req, '_session') if hasattr(req, '_user_info'): - ## For the same reason we can delete the user_info. + # For the same reason we can delete the user_info. delattr(req, '_user_info') for (callback, data) in req.get_cleanups(): callback(data) - ## as suggested in - ## + # as suggested in + # gc.enable() gc.collect() del gc.garbage[:] return [] + def generate_error_page(req, admin_was_alerted=True, page_already_started=False): - """ - Returns an iterable with the error page to be sent to the user browser. - """ + """Return an iterable with the error page to be sent to the browser.""" from invenio.webpage import page from invenio import template webstyle_templates = template.load('webstyle') @@ -608,9 +623,11 @@ def generate_error_page(req, admin_was_alerted=True, page_already_started=False) else: return [page(title=req.get_wsgi_status(), body=webstyle_templates.tmpl_error_page(status=req.get_wsgi_status(), ln=ln, admin_was_alerted=admin_was_alerted), language=ln, req=req)] + def is_static_path(path): """ - Returns True if path corresponds to an exsting file under CFG_WEBDIR. + Return True if path corresponds to an exsting file under CFG_WEBDIR. + @param path: the path. @type path: string @return: True if path corresponds to an exsting file under CFG_WEBDIR. @@ -621,9 +638,10 @@ def is_static_path(path): return path return None + def is_mp_legacy_publisher_path(path): - """ - Checks path corresponds to an exsting Python file under CFG_WEBDIR. + """Check path corresponds to an exsting Python file under CFG_WEBDIR. + @param path: the path. @type path: string @return: the path of the module to load and the function to call there. @@ -643,36 +661,35 @@ def is_mp_legacy_publisher_path(path): else: return None, None + def mp_legacy_publisher(req, possible_module, possible_handler): - """ - mod_python legacy publisher minimum implementation. - """ + """mod_python legacy publisher minimum implementation.""" the_module = open(possible_module).read() module_globals = {} exec(the_module, module_globals) if possible_handler in module_globals and callable(module_globals[possible_handler]): from invenio.webinterface_handler import _check_result - ## req is the required first parameter of any handler + # req is the required first parameter of any handler expected_args = list(inspect.getargspec(module_globals[possible_handler])[0]) if not expected_args or 'req' != expected_args[0]: - ## req was not the first argument. Too bad! + # req was not the first argument. Too bad! raise SERVER_RETURN, HTTP_NOT_FOUND - ## the req.form must be casted to dict because of Python 2.4 and earlier - ## otherwise any object exposing the mapping interface can be - ## used with the magic ** + # the req.form must be casted to dict because of Python 2.4 and earlier + # otherwise any object exposing the mapping interface can be + # used with the magic ** form = dict(req.form) for key, value in form.items(): - ## FIXME: this is a backward compatibility workaround - ## because most of the old administration web handler - ## expect parameters to be of type str. - ## When legacy publisher will be removed all this - ## pain will go away anyway :-) + # FIXME: this is a backward compatibility workaround + # because most of the old administration web handler + # expect parameters to be of type str. + # When legacy publisher will be removed all this + # pain will go away anyway :-) if isinstance(value, str): form[key] = str(value) else: - ## NOTE: this is a workaround for e.g. legacy webupload - ## that is still using legacy publisher and expect to - ## have a file (Field) instance instead of a string. + # NOTE: this is a workaround for e.g. legacy webupload + # that is still using legacy publisher and expect to + # have a file (Field) instance instead of a string. form[key] = value if (CFG_FULL_HTTPS or CFG_HAS_HTTPS_SUPPORT and get_session(req).need_https) and not req.is_https(): @@ -686,7 +703,7 @@ def mp_legacy_publisher(req, possible_module, possible_handler): # Compute the new path plain_path = original_parts[2] plain_path = secure_prefix_parts[2] + \ - plain_path[len(plain_prefix_parts[2]):] + plain_path[len(plain_prefix_parts[2]):] # ...and recompose the complete URL final_parts = list(secure_prefix_parts) @@ -720,6 +737,7 @@ def mp_legacy_publisher(req, possible_module, possible_handler): else: raise SERVER_RETURN, HTTP_NOT_FOUND + def check_wsgiref_testing_feasability(): """ In order to use wsgiref for running Invenio, CFG_SITE_URL and @@ -738,10 +756,9 @@ def check_wsgiref_testing_feasability(): Currently CFG_SITE_SECURE_URL is set to: "%s".""" % CFG_SITE_SECURE_URL sys.exit(1) + def wsgi_handler_test(port=80): - """ - Simple WSGI testing environment based on wsgiref. - """ + """Simple WSGI testing environment based on wsgiref.""" from wsgiref.simple_server import make_server global CFG_WSGI_SERVE_STATIC_FILES CFG_WSGI_SERVE_STATIC_FILES = True @@ -751,6 +768,7 @@ def wsgi_handler_test(port=80): print "Serving on port %s..." % port httpd.serve_forever() + def main(): from optparse import OptionParser parser = OptionParser() From 3d3798f75a4dadb6a4e6865ce3b1d20d752285ed Mon Sep 17 00:00:00 2001 From: "Esteban J. G. Gabancho" Date: Tue, 14 Jul 2015 17:04:50 +0200 Subject: [PATCH 39/59] WebSubmit: subtitle file addition to converter --- modules/websubmit/lib/websubmit_file_converter.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/websubmit/lib/websubmit_file_converter.py b/modules/websubmit/lib/websubmit_file_converter.py index 75255e62b3..25a52d12bc 100644 --- a/modules/websubmit/lib/websubmit_file_converter.py +++ b/modules/websubmit/lib/websubmit_file_converter.py @@ -138,6 +138,8 @@ def get_conversion_map(): '.hocr': {}, '.pdf;pdfa': {}, '.asc': {}, + '.vtt': {}, + '.srt': {}, } if CFG_PATH_GZIP: ret['.ps']['.ps.gz'] = (gzip, {}) @@ -186,6 +188,9 @@ def get_conversion_map(): ret['.html']['.txt'] = (html2text, {}) ret['.htm']['.txt'] = (html2text, {}) ret['.xml']['.txt'] = (html2text, {}) + # Subtitle files + ret['.vtt']['.txt'] = (txt2text, {}) + ret['.srt']['.txt'] = (txt2text, {}) if CFG_PATH_TIFF2PDF: ret['.tiff']['.pdf'] = (tiff2pdf, {}) ret['.tif']['.pdf'] = (tiff2pdf, {}) From f97834471cacfffa6633c5c9e06f36eba4659fd1 Mon Sep 17 00:00:00 2001 From: "Esteban J. G. Gabancho" Date: Tue, 14 Jul 2015 17:05:05 +0200 Subject: [PATCH 40/59] BibIndex: external file index enhancement Signed-off-by: Esteban J. G. Gabancho --- config/invenio.conf | 2 ++ .../tokenizers/BibIndexFulltextTokenizer.py | 36 +++++++++++-------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/config/invenio.conf b/config/invenio.conf index 408411ba6a..bca923df90 100644 --- a/config/invenio.conf +++ b/config/invenio.conf @@ -1171,6 +1171,8 @@ CFG_BIBINDEX_PERFORM_OCR_ON_DOCNAMES = scan-.* # NOTE: for backward compatibility reasons you can set this to a simple # regular expression that will directly be used as the unique key of the # map, with corresponding value set to ".*" (in order to match any URL) +# NOTE2: If the value is None, the url mapping the key regex will be used +# directly CFG_BIBINDEX_SPLASH_PAGES = { "http://documents\.cern\.ch/setlink\?.*": ".*", "http://ilcagenda\.linearcollider\.org/subContributionDisplay\.py\?.*|http://ilcagenda\.linearcollider\.org/contributionDisplay\.py\?.*": "http://ilcagenda\.linearcollider\.org/getFile\.py/access\?.*|http://ilcagenda\.linearcollider\.org/materialDisplay\.py\?.*", diff --git a/modules/bibindex/lib/tokenizers/BibIndexFulltextTokenizer.py b/modules/bibindex/lib/tokenizers/BibIndexFulltextTokenizer.py index c26dbdfbcb..3f2b04a582 100644 --- a/modules/bibindex/lib/tokenizers/BibIndexFulltextTokenizer.py +++ b/modules/bibindex/lib/tokenizers/BibIndexFulltextTokenizer.py @@ -43,6 +43,7 @@ from invenio.bibtask import write_message from invenio.errorlib import register_exception from invenio.intbitset import intbitset +from invenio.search_engine import search_pattern from invenio.bibindex_tokenizers.BibIndexDefaultTokenizer import BibIndexDefaultTokenizer @@ -131,16 +132,18 @@ def get_words_from_fulltext(self, url_direct_or_indirect): for splash_re, url_re in CFG_BIBINDEX_SPLASH_PAGES.iteritems(): if re.match(splash_re, url_direct_or_indirect): write_message("... %s is a splash page (%s)" % (url_direct_or_indirect, splash_re), verbose=2) - html = urllib2.urlopen(url_direct_or_indirect).read() - urls = get_links_in_html_page(html) - write_message("... found these URLs in %s splash page: %s" % (url_direct_or_indirect, ", ".join(urls)), verbose=3) - for url in urls: - if re.match(url_re, url): - write_message("... will index %s (matched by %s)" % (url, url_re), verbose=2) - urls_to_index.add(url) - if not urls_to_index: - urls_to_index.add(url_direct_or_indirect) - write_message("... will extract words from %s" % ', '.join(urls_to_index), verbose=2) + if url_re is None: + urls_to_index.add(url_direct_or_indirect) + continue + else: + html = urllib2.urlopen(url_direct_or_indirect).read() + urls = get_links_in_html_page(html) + write_message("... found these URLs in %s splash page: %s" % (url_direct_or_indirect, ", ".join(urls)), verbose=3) + for url in urls: + if re.match(url_re, url): + write_message("... will index %s (matched by %s)" % (url, url_re), verbose=2) + urls_to_index.add(url) + write_message("... will extract words from {0}".format(urls_to_index), verbose=2) words = {} for url in urls_to_index: tmpdoc = download_url(url) @@ -156,11 +159,14 @@ def get_words_from_fulltext(self, url_direct_or_indirect): indexer = get_idx_indexer('fulltext') if indexer != 'native': - if indexer == 'SOLR' and CFG_SOLR_URL: - solr_add_fulltext(None, text) # FIXME: use real record ID - if indexer == 'XAPIAN' and CFG_XAPIAN_ENABLED: - #xapian_add(None, 'fulltext', text) # FIXME: use real record ID - pass + recids = search_pattern(p='8567_u:{0}'.format(url)) + write_message('... will add words to record {0}'.format(recid), verbose=2) + for recid in recids: + if indexer == 'SOLR' and CFG_SOLR_URL: + solr_add_fulltext(recid, text) + if indexer == 'XAPIAN' and CFG_XAPIAN_ENABLED: + #xapian_add(recid, 'fulltext', text) + pass # we are relying on an external information retrieval system # to provide full-text indexing, so dispatch text to it and # return nothing here: From 0e48dc25e42f0f58e1039474dd0726eeb523d8da Mon Sep 17 00:00:00 2001 From: "Esteban J. G. Gabancho" Date: Wed, 15 Jul 2015 13:37:46 +0200 Subject: [PATCH 41/59] BibRank: allow ranking external files using solr Signed-off-by: Esteban J. G. Gabancho --- .../miscutil/lib/solrutils_bibrank_indexer.py | 81 ++++++++++++++++--- 1 file changed, 68 insertions(+), 13 deletions(-) diff --git a/modules/miscutil/lib/solrutils_bibrank_indexer.py b/modules/miscutil/lib/solrutils_bibrank_indexer.py index fd0f83da19..06b45cb03d 100644 --- a/modules/miscutil/lib/solrutils_bibrank_indexer.py +++ b/modules/miscutil/lib/solrutils_bibrank_indexer.py @@ -22,17 +22,32 @@ """ +import os +import urllib2 +import re import time -from invenio.config import CFG_SOLR_URL + +from invenio.config import ( + CFG_SOLR_URL, + CFG_BIBINDEX_FULLTEXT_INDEX_LOCAL_FILES_ONLY, + CFG_BIBINDEX_SPLASH_PAGES +) from invenio.bibtask import write_message, task_get_option, task_update_progress, \ task_sleep_now_if_required +from invenio.htmlutils import get_links_in_html_page +from invenio.websubmit_file_converter import convert_file from invenio.dbquery import run_sql -from invenio.search_engine import record_exists -from invenio.bibdocfile import BibRecDocs +from invenio.search_engine import record_exists, get_field_tags +from invenio.search_engine_utils import get_fieldvalues +from invenio.bibdocfile import BibRecDocs, bibdocfile_url_p, download_url from invenio.solrutils_bibindex_indexer import replace_invalid_solr_characters from invenio.bibindex_engine import create_range_list from invenio.errorlib import register_exception from invenio.bibrank_bridge_utils import get_tags, get_field_content_in_utf8 +from invenio.bibtask import write_message + + +SOLR_CONNECTION = None if CFG_SOLR_URL: @@ -103,16 +118,11 @@ def solr_add_range(lower_recid, upper_recid, tags_to_index, next_commit_counter) """ for recid in range(lower_recid, upper_recid + 1): if record_exists(recid): - abstract = get_field_content_in_utf8(recid, 'abstract', tags_to_index) - author = get_field_content_in_utf8(recid, 'author', tags_to_index) - keyword = get_field_content_in_utf8(recid, 'keyword', tags_to_index) - title = get_field_content_in_utf8(recid, 'title', tags_to_index) - try: - bibrecdocs = BibRecDocs(recid) - fulltext = unicode(bibrecdocs.get_text(), 'utf-8') - except: - fulltext = '' - + abstract = get_field_content_in_utf8(recid, 'abstract', tags_to_index) + author = get_field_content_in_utf8(recid, 'author', tags_to_index) + keyword = get_field_content_in_utf8(recid, 'keyword', tags_to_index) + title = get_field_content_in_utf8(recid, 'title', tags_to_index) + fulltext = _get_fulltext(recid) solr_add(recid, abstract, author, fulltext, keyword, title) next_commit_counter = solr_commit_if_necessary(next_commit_counter,recid=recid) @@ -182,3 +192,48 @@ def word_index(run): # pylint: disable=W0613 write_message("No new records. Solr index is up to date") write_message("Solr ranking indexer completed") + + +def _get_fulltext(recid): + + words = [] + try: + bibrecdocs = BibRecDocs(recid) + words.append(unicode(bibrecdocs.get_text(), 'utf-8')) + except Exception, e: + pass + + if CFG_BIBINDEX_FULLTEXT_INDEX_LOCAL_FILES_ONLY: + write_message("... %s is external URL but indexing only local files" % url, verbose=2) + return ' '.join(words) + + urls_from_record = [url for tag in get_field_tags('fulltext') + for url in get_fieldvalues(recid, tag) + if not bibdocfile_url_p(url)] + urls_to_index = set() + for url_direct_or_indirect in urls_from_record: + for splash_re, url_re in CFG_BIBINDEX_SPLASH_PAGES.iteritems(): + if re.match(splash_re, url_direct_or_indirect): + if url_re is None: + write_message("... %s is file to index (%s)" % (url_direct_or_indirect, splash_re), verbose=2) + urls_to_index.add(url_direct_or_indirect) + continue + write_message("... %s is a splash page (%s)" % (url_direct_or_indirect, splash_re), verbose=2) + html = urllib2.urlopen(url_direct_or_indirect).read() + urls = get_links_in_html_page(html) + write_message("... found these URLs in %s splash page: %s" % (url_direct_or_indirect, ", ".join(urls)), verbose=3) + for url in urls: + if re.match(url_re, url): + write_message("... will index %s (matched by %s)" % (url, url_re), verbose=2) + urls_to_index.add(url) + if urls_to_index: + write_message("... will extract words from %s:%s" % (recid, ', '.join(urls_to_index)), verbose=2) + for url in urls_to_index: + tmpdoc = download_url(url) + try: + tmptext = convert_file(tmpdoc, output_format='.txt') + words.append(open(tmptext).read()) + os.remove(tmptext) + finally: + os.remove(tmpdoc) + return ' '.join(words) From 99266246e9049b35ac7a5a23fbc19cb6e71951b6 Mon Sep 17 00:00:00 2001 From: Flavio Costa Date: Fri, 17 Jul 2015 14:11:43 +0200 Subject: [PATCH 42/59] WebJournal: navigation menu category replacement * Changes in main navigation menu: Training->Learning --- .../lib/elements/bfe_webjournal_main_navigation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/webjournal/lib/elements/bfe_webjournal_main_navigation.py b/modules/webjournal/lib/elements/bfe_webjournal_main_navigation.py index 6ac3a512c2..0de42bc293 100644 --- a/modules/webjournal/lib/elements/bfe_webjournal_main_navigation.py +++ b/modules/webjournal/lib/elements/bfe_webjournal_main_navigation.py @@ -20,7 +20,7 @@ """ WebJournal element - Prints main (horizontal) navigation menu """ -from invenio.webjournal_utils import \ +from invenio.legacy.webjournal_utils import \ parse_url_string, \ make_journal_url, \ get_journal_categories @@ -66,7 +66,7 @@ def format_element(bfo, category_prefix, category_suffix, separator=" | ", linkattrd = {'class':'selectedNavigationPage'} if journal_name == 'CERNBulletin' and \ category == 'Training and Development': - category = 'Training' + category = 'Learning' if ln == 'fr': category = 'Formations' category_link = create_html_link(category_url, {}, @@ -96,6 +96,6 @@ def escape_values(bfo): dummy = _("Training and Development") dummy = _("General Information") dummy = _("Announcements") -dummy = _("Training") +dummy = _("Learning") dummy = _("Events") dummy = _("Staff Association") From 78ab9370d05a2e81c8400f7fd7313ccf9865dcc1 Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Fri, 21 Dec 2012 16:18:36 +0100 Subject: [PATCH 43/59] WebNews: New module to display and manage news * The WebNews module that will be responsible for presenting the site's news to the users and keeping a history of them. * This first release includes the tooltip functionality and the database structure. (addresses #1288) --- configure.ac | 4 + modules/Makefile.am | 1 + ...nvenio_2013_02_15_webnews_new_db_tables.py | 82 +++++ modules/webnews/Makefile.am | 20 ++ modules/webnews/doc/Makefile.am | 18 ++ modules/webnews/lib/Makefile.am | 36 +++ modules/webnews/lib/webnews.css | 162 ++++++++++ modules/webnews/lib/webnews.js | 255 +++++++++++++++ modules/webnews/lib/webnews.py | 291 ++++++++++++++++++ modules/webnews/lib/webnews_config.py | 34 ++ modules/webnews/lib/webnews_dblayer.py | 98 ++++++ modules/webnews/lib/webnews_utils.py | 61 ++++ modules/webnews/lib/webnews_webinterface.py | 81 +++++ modules/webnews/web/Makefile.am | 18 ++ modules/webstyle/lib/webinterface_layout.py | 10 +- modules/webstyle/lib/webstyle_templates.py | 28 ++ 16 files changed, 1198 insertions(+), 1 deletion(-) create mode 100644 modules/miscutil/lib/upgrades/invenio_2013_02_15_webnews_new_db_tables.py create mode 100644 modules/webnews/Makefile.am create mode 100644 modules/webnews/doc/Makefile.am create mode 100644 modules/webnews/lib/Makefile.am create mode 100644 modules/webnews/lib/webnews.css create mode 100644 modules/webnews/lib/webnews.js create mode 100644 modules/webnews/lib/webnews.py create mode 100644 modules/webnews/lib/webnews_config.py create mode 100644 modules/webnews/lib/webnews_dblayer.py create mode 100644 modules/webnews/lib/webnews_utils.py create mode 100644 modules/webnews/lib/webnews_webinterface.py create mode 100644 modules/webnews/web/Makefile.am diff --git a/configure.ac b/configure.ac index f634d004fd..24162dba6b 100644 --- a/configure.ac +++ b/configure.ac @@ -874,6 +874,10 @@ AC_CONFIG_FILES([config.nice \ modules/webmessage/doc/hacking/Makefile \ modules/webmessage/lib/Makefile \ modules/webmessage/web/Makefile \ + modules/webnews/Makefile \ + modules/webnews/doc/Makefile \ + modules/webnews/lib/Makefile \ + modules/webnews/web/Makefile \ modules/websearch/Makefile \ modules/websearch/bin/Makefile \ modules/websearch/bin/webcoll \ diff --git a/modules/Makefile.am b/modules/Makefile.am index 0d72e5de03..469ab7c02b 100644 --- a/modules/Makefile.am +++ b/modules/Makefile.am @@ -52,6 +52,7 @@ SUBDIRS = bibauthorid \ webjournal \ weblinkback \ webmessage \ + webnews \ websearch \ websession \ webstat \ diff --git a/modules/miscutil/lib/upgrades/invenio_2013_02_15_webnews_new_db_tables.py b/modules/miscutil/lib/upgrades/invenio_2013_02_15_webnews_new_db_tables.py new file mode 100644 index 0000000000..bc01d22971 --- /dev/null +++ b/modules/miscutil/lib/upgrades/invenio_2013_02_15_webnews_new_db_tables.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +## +## This file is part of Invenio. +## Copyright (C) 2012 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +from invenio.dbquery import run_sql + +depends_on = ['invenio_release_1_1_0'] + +def info(): + return "New database tables for the WebNews module" + +def do_upgrade(): + query_story = \ +"""DROP TABLE IF EXISTS `nwsSTORY`; +CREATE TABLE `nwsSTORY` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `title` varchar(256) NOT NULL, + `body` text NOT NULL, + `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +);""" + run_sql(query_story) + + query_tag = \ +"""DROP TABLE IF EXISTS `nwsTAG`; +CREATE TABLE `nwsTAG` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tag` varchar(64) NOT NULL, + PRIMARY KEY (`id`) +);""" + run_sql(query_tag) + + query_tooltip = \ +"""DROP TABLE IF EXISTS `nwsTOOLTIP`; +CREATE TABLE `nwsTOOLTIP` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `id_story` int(11) NOT NULL, + `body` varchar(512) NOT NULL, + `target_element` varchar(256) NOT NULL DEFAULT '', + `target_page` varchar(256) NOT NULL DEFAULT '', + PRIMARY KEY (`id`), + KEY `id_story` (`id_story`), + CONSTRAINT `nwsTOOLTIP_ibfk_1` FOREIGN KEY (`id_story`) REFERENCES `nwsSTORY` (`id`) +);""" + run_sql(query_tooltip) + + query_story_tag = \ +"""DROP TABLE IF EXISTS `nwsSTORY_nwsTAG`; +CREATE TABLE `nwsSTORY_nwsTAG` ( + `id_story` int(11) NOT NULL, + `id_tag` int(11) NOT NULL, + PRIMARY KEY (`id_story`,`id_tag`), + KEY `id_story` (`id_story`), + KEY `id_tag` (`id_tag`), + CONSTRAINT `nwsSTORY_nwsTAG_ibfk_1` FOREIGN KEY (`id_story`) REFERENCES `nwsSTORY` (`id`), + CONSTRAINT `nwsSTORY_nwsTAG_ibfk_2` FOREIGN KEY (`id_tag`) REFERENCES `nwsTAG` (`id`) +);""" + run_sql(query_story_tag) + +def estimate(): + return 1 + +def pre_upgrade(): + pass + +def post_upgrade(): + pass diff --git a/modules/webnews/Makefile.am b/modules/webnews/Makefile.am new file mode 100644 index 0000000000..b75d18399a --- /dev/null +++ b/modules/webnews/Makefile.am @@ -0,0 +1,20 @@ +## This file is part of Invenio. +## Copyright (C) 2005, 2006, 2007, 2008, 2010, 2011 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +SUBDIRS = lib web doc + +CLEANFILES = *~ diff --git a/modules/webnews/doc/Makefile.am b/modules/webnews/doc/Makefile.am new file mode 100644 index 0000000000..fb4b48b5c1 --- /dev/null +++ b/modules/webnews/doc/Makefile.am @@ -0,0 +1,18 @@ +## This file is part of Invenio. +## Copyright (C) 2005, 2006, 2007, 2008, 2010, 2011 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +CLEANFILES = *~ diff --git a/modules/webnews/lib/Makefile.am b/modules/webnews/lib/Makefile.am new file mode 100644 index 0000000000..0059e32654 --- /dev/null +++ b/modules/webnews/lib/Makefile.am @@ -0,0 +1,36 @@ +## This file is part of Invenio. +## Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +pylibdir = $(libdir)/python/invenio +webdir = $(localstatedir)/www/img +jsdir = $(localstatedir)/www/js + +pylib_DATA = webnews.py \ + webnews_config.py \ + webnews_webinterface.py \ + webnews_dblayer.py \ + webnews_utils.py + +js_DATA = webnews.js + +web_DATA = webnews.css + +EXTRA_DIST = $(pylib_DATA) \ + $(js_DATA) \ + $(web_DATA) + +CLEANFILES = *~ *.tmp *.pyc diff --git a/modules/webnews/lib/webnews.css b/modules/webnews/lib/webnews.css new file mode 100644 index 0000000000..b6de6eca33 --- /dev/null +++ b/modules/webnews/lib/webnews.css @@ -0,0 +1,162 @@ +/*************************************************************************** +* Assume that we want the arrow's size (i.e. side; height in case it's on * +* the left or right side; width otherwise) to be 16px. Then the border * +* width for the arrow and its border will be half that dimension, 8px. In * +* order for the arrow to be right on the tooltip, the arrow border's left * +* positioning has to be as much as it's size, therefore -16px. The arrow's * +* left positioning has to be a couple of pixels to the right in order for * +* the border effect to be visible, therefore -14px. At the same time, in * +* order for the arrow's point to not overlap with the targeted item, the * +* tooltip's left margin has to be half the arrow's size (=the arrow's * +* border width), 8px. The tooltip's padding has to at least as much as the * +* arrow's left positioning overhead, (-(-16px-(-14px))) 2px in this case. * +* The arrow's and the arrow border's top positioning can be as much as we * +* want, and it must be the same for both. * +* * +* Desired arrow's size ([height|width]): 16px * +* Arrow's border-width: 8px (half the arrow's size) * +* Tooltip's [left|right] margin: 8px (half the arrow's size) * +* Arrow's [left|right] positioning: -16px & -14px (arrow border and arrow) * +* Arrow's [top|bottom] positioning: 8px * +* * +***************************************************************************/ + +/* Arrow implementation using traditional elements */ +.ttn { + /* Display */ + position: absolute; + display: none; + z-index: 9997; + + /* Dimensions */ + max-width: 250px; + min-width: 150px; + + /* Contents */ + background: #FFFFCC; + margin-left: 8px; + padding: 5px; /* padding has to be at least 2 */ + font-size: small; + + /* Border */ + border: 1px solid #FFCC00; + -moz-border-radius: 5px; + -webkit-border-radius: 5px; + -o-border-radius: 5px; + border-radius: 5px; + + /* Shadow */ + -moz-box-shadow: 1px 1px 3px gray; + -webkit-box-shadow: 1px 1px 3px gray; + -o-box-shadow: 1px 1px 3px gray; + box-shadow: 1px 1px 3px gray; +} + +.ttn_arrow { + /* Display */ + position: absolute; + left: -14px; + top: 8px; + z-index: 9999; + + /* Dimensions */ + height: 0; + width: 0; + + /* Border */ + border-color: transparent #FFFFCC transparent transparent; + border-color: rgba(255,255,255,0) #FFFFCC rgba(255,255,255,0) rgba(255,255,255,0); + border-style: solid; + border-width: 8px; +} + +.ttn_arrow_border { + /* Display */ + position: absolute; + left: -16px; + top: 8px; + z-index: 9998; + + /* Dimensions */ + height: 0; + width: 0; + + /* Border */ + border-color: transparent #FFCC00 transparent transparent; + border-color: rgba(255,255,255,0) #FFCC00 rgba(255,255,255,0) rgba(255,255,255,0); + border-style: solid; + border-width: 8px; +} + +/* Arrow implementation using the :before and :after pseudo elements */ +.ttn_alt_arrow_box { + /* Display */ + position: absolute; + display: none; + z-index: 9997; + + /* Dimensions */ + max-width: 250px; + min-width: 150px; + + /* Contents */ + background: #FFFFCC; + margin-left: 6px; + padding: 3px; + + /* Border */ + border: 1px solid #FFCC00; + -moz-border-radius: 5px; + -webkit-border-radius: 5px; + -o-border-radius: 5px; + border-radius: 5px; + + /* Shadow */ + -moz-box-shadow: 1px 1px 3px gray; + -webkit-box-shadow: 1px 1px 3px gray; + -o-box-shadow: 1px 1px 3px gray; + box-shadow: 1px 1px 3px gray; +} + +.ttn_alt_arrow_box:after, .ttn_alt_arrow_box:before { + right: 100%; + border: 1px solid transparent; + content: " "; + height: 0; + width: 0; + position: absolute; + pointer-events: none; +} + +.ttn_alt_arrow_box:after { + border-color: transparent; + border-right-color: #FFFFCC; + border-width: 6px; + top: 13px; +} + +.ttn_alt_arrow_box:before { + border-color: transparent; + border-right-color: #FFCC00; + border-width: 8px; + top: 11px; +} + +/* Style for the tooltip's contents */ +.ttn_text { + /* Contents */ + vertical-align: top; + text-align: left; +} + +.ttn_actions { + /* Contents */ + vertical-align: bottom; + text-align: right; +} + +.ttn_actions_read_more { +} + +.ttn_actions_dismiss { +} diff --git a/modules/webnews/lib/webnews.js b/modules/webnews/lib/webnews.js new file mode 100644 index 0000000000..932d8b18b2 --- /dev/null +++ b/modules/webnews/lib/webnews.js @@ -0,0 +1,255 @@ +/****************************************************************************** +* WebNews JavaScript Library +* +* Includes functions to create and place the tooltips, and re-place them +* in case the browser window is resized. +* +* TODO: remove magic numbers and colors +* +******************************************************************************/ + +// Tooltips entry function. Create all the tooltips. +function create_tooltips(data) { + + // Get all the tooltips. + var tooltips = data['tooltips']; + + // Only proceed if there are tooltips. + if ( tooltips != undefined ) { + + // Get the story id and ln, to be used to create the "Read more" URL. + var story_id = data['story_id']; + var ln = data['ln']; + + // Keep an array of the tooltip notification and target elements, + // to be used to speed up the window resize function later. + var tooltips_elements = []; + + // Create each tooltip and get its notification and target elements. + for (var i = 0; i < tooltips.length; i++) { + tooltip = tooltips[i]; + tooltip_elements = create_tooltip(tooltip, story_id, ln); + tooltips_elements.push(tooltip_elements); + } + + /* + // To cover most cases, we need to call re_place_tooltip both on page + // resize and page scroll. So, let's combine both events usings ".on()" + $(window).resize(function() { + for (var i = 0; i < tooltips_elements.length; i++) { + var tooltip_notification = tooltips_elements[i][0]; + var tooltip_target = tooltips_elements[i][1]; + re_place_tooltip(tooltip_notification, tooltip_target); + } + }); + + $(window).scroll(function() { + for (var i = 0; i < tooltips_elements.length; i++) { + var tooltip_notification = tooltips_elements[i][0]; + var tooltip_target = tooltips_elements[i][1]; + re_place_tooltip(tooltip_notification, tooltip_target); + } + }); + */ + + $(window).on("resize scroll", function() { + for (var i = 0; i < tooltips_elements.length; i++) { + var tooltip_notification = tooltips_elements[i][0]; + var tooltip_target = tooltips_elements[i][1]; + re_place_tooltip(tooltip_notification, tooltip_target); + } + }); + + } + +} + +function create_tooltip(tooltip, story_id, ln) { + + // Get the tooltip data. + var id = tooltip['id']; + var target = tooltip['target']; + var body = tooltip['body']; + var readmore = tooltip['readmore']; + var dismiss = tooltip['dismiss']; + + // Create the "Read more" URL. + var readmore_url = '/news/story?id=' + story_id + '&ln=' + ln + + // Construct the tooltip html. + var tooltip_html = '
    \n'; + tooltip_html += '
    ' + body + '
    \n'; + tooltip_html += '
    \n'; + // TODO: Do not add the "Read more" label until the /news interface is ready. + //tooltip_html += ' ' + readmore + '\n'; + //tooltip_html += '  | \n'; + tooltip_html += ' ' + dismiss + '\n'; + tooltip_html += '
    \n'; + tooltip_html += '
    \n'; + tooltip_html += '
    \n'; + tooltip_html += '
    \n'; + + // Append the tooltip html to the body. + $('body').append(tooltip_html); + + // Create the jquery element selectors for the tooltip notification and target. + var tooltip_notification = $("#" + id); + var tooltip_target = eval(target); + + // Place and display the tooltip. + place_tooltip(tooltip_notification, tooltip_target); + + // Return the tooltip notification and target elements in an array. + return [tooltip_notification, tooltip_target]; +} + +function place_tooltip(tooltip_notification, tooltip_target) { + + // Only display the tooltip if the tooltip_notification exists + // and the tooltip exists and is visible. + if ( tooltip_notification.length > 0 && tooltip_target.length > 0 && tooltip_target.is(":visible") ) { + + // First, calculate the top of tooltip_notification: + // This comes from tooltip_target's top with some adjustments + // in order to place the tooltip in the middle of the tooltip_target. + var tooltip_target_height = tooltip_target.outerHeight(); + var tooltip_target_top = tooltip_target.offset().top; + // The distance from the top of the tooltip_notifcation to the + // arrow's tip is 16px (half the arrow's size + arrow's top margin) + var tooltip_notification_top = tooltip_target_top + ( tooltip_target_height / 2 ) - 16 + if ( tooltip_notification_top < 0 ) { + tooltip_notification_top = 0; + } + + // Second, calculate the left of tooltip_notification: + // This comes from the sum of tooltip_target's left and width + var tooltip_target_left = tooltip_target.offset().left; + var tooltip_target_width = tooltip_target.outerWidth(); + var tooltip_notification_left = tooltip_target_left + tooltip_target_width; + + // However, if tooltip_notification appears to be displayed outside the window, + // then we have to place it on the other side of tooltip_target + var tooltip_notification_width = tooltip_notification.outerWidth(); + var window_width = $(window).width(); + if ( ( tooltip_notification_left + tooltip_notification_width ) > window_width ) { + // Place tooltip_notification on the other side, taking into account the arrow's size + // The left margin of the tooltip_notification and half the + // arrow's size is 16px + tooltip_notification_left = tooltip_target_left - tooltip_notification_width - 16; + // Why does 4px work perfectly here? + tooltip_notification.children("div[class='ttn_arrow']").css("left", (tooltip_notification_width - 4) + "px"); + tooltip_notification.children("div[class='ttn_arrow']").css("border-color", "transparent transparent transparent #FFFFCC"); + tooltip_notification.children("div[class='ttn_arrow']").css("border-color", "rgba(255,255,255,0) rgba(255,255,255,0) rgba(255,255,255,0) #FFFFCC"); + // 2px is 4px - 2px here, since the arrow's border has a 2px offset from the arrow + tooltip_notification.children("div[class='ttn_arrow_border']").css("left", "").css("left", (tooltip_notification_width - 2) + "px"); + tooltip_notification.children("div[class='ttn_arrow_border']").css("border-color", "transparent transparent transparent #FFCC00"); + tooltip_notification.children("div[class='ttn_arrow_border']").css("border-color", "rgba(255,255,255,0) rgba(255,255,255,0) rgba(255,255,255,0) #FFCC00"); + tooltip_notification.css("-moz-box-shadow", "-1px 1px 3px gray"); + tooltip_notification.css("-webkit-box-shadow", "-1px 1px 3px gray"); + tooltip_notification.css("-o-box-shadow", "-1px 1px 3px gray"); + tooltip_notification.css("box-shadow", "-1px 1px 3px gray"); + } + + // Set the final attributes and display tooltip_notification + tooltip_notification.css('top', tooltip_notification_top + 'px'); + tooltip_notification.css('left', tooltip_notification_left + 'px'); + tooltip_notification.fadeIn(); + tooltip_notification.find("a[class='ttn_actions_dismiss']").click(function() { + $.ajax({ + url: "/news/dismiss", + data: { tooltip_notification_id: tooltip_notification.attr("id") }, + success: function(data) { + if ( data["success"] == 1 ) { + tooltip_notification.fadeOut(); + } + }, + dataType: "json" + }); + }); + + } + +} + +function re_place_tooltip(tooltip_notification, tooltip_target) { + + // Only display the tooltip if the tooltip_notification exists + // and the tooltip exists and is visible. + if ( tooltip_notification.length > 0 && tooltip_notification.is(":visible") && tooltip_target.length > 0 && tooltip_target.is(":visible") ) { + + // First, calculate the top of tooltip_notification: + // This comes from tooltip_target's top with some adjustments + // in order to place the tooltip in the middle of the tooltip_target. + var tooltip_target_height = tooltip_target.outerHeight(); + var tooltip_target_top = tooltip_target.offset().top; + // The distance from the top of the tooltip_notifcation to the + // arrow's tip is 16px (half the arrow's size + arrow's top margin) + var tooltip_notification_top = tooltip_target_top + ( tooltip_target_height / 2 ) - 16 + if ( tooltip_notification_top < 0 ) { + tooltip_notification_top = 0; + } + + // Second, calculate the left of tooltip_notification: + // This comes from the sum of tooltip_target's left and width + var tooltip_target_left = tooltip_target.offset().left; + var tooltip_target_width = tooltip_target.outerWidth(); + var tooltip_notification_left = tooltip_target_left + tooltip_target_width; + + // However, if tooltip_notification appears to be displayed outside the window, + // then we have to place it on the other side of tooltip_target + var tooltip_notification_width = tooltip_notification.outerWidth(); + var window_width = $(window).width(); + if ( ( tooltip_notification_left + tooltip_notification_width ) > window_width ) { + // Place tooltip_notification on the other side, taking into account the arrow's size + // The left margin of the tooltip_notification and half the + // arrow's size is 16px + tooltip_notification_left = tooltip_target_left - tooltip_notification_width - 16; + // Why does 4px work perfectly here? + tooltip_notification.children("div[class='ttn_arrow']").css("left", (tooltip_notification_width - 4) + "px"); + tooltip_notification.children("div[class='ttn_arrow']").css("border-color", "transparent transparent transparent #FFFFCC"); + tooltip_notification.children("div[class='ttn_arrow']").css("border-color", "rgba(255,255,255,0) rgba(255,255,255,0) rgba(255,255,255,0) #FFFFCC"); + // 2px is 4px - 2px here, since the arrow's border has a 2px offset from the arrow + tooltip_notification.children("div[class='ttn_arrow_border']").css("left", "").css("left", (tooltip_notification_width - 2) + "px"); + tooltip_notification.children("div[class='ttn_arrow_border']").css("border-color", "transparent transparent transparent #FFCC00"); + tooltip_notification.children("div[class='ttn_arrow_border']").css("border-color", "rgba(255,255,255,0) rgba(255,255,255,0) rgba(255,255,255,0) #FFCC00"); + tooltip_notification.css("-moz-box-shadow", "-1px 1px 3px gray"); + tooltip_notification.css("-webkit-box-shadow", "-1px 1px 3px gray"); + tooltip_notification.css("-o-box-shadow", "-1px 1px 3px gray"); + tooltip_notification.css("box-shadow", "-1px 1px 3px gray"); + } + else { + // The original left position of the tooltip_notification's arrow is -14px + tooltip_notification.children("div[class='ttn_arrow']").css("left", "-14px"); + tooltip_notification.children("div[class='ttn_arrow']").css("border-color", "transparent #FFFFCC transparent transparent"); + tooltip_notification.children("div[class='ttn_arrow']").css("border-color", "rgba(255,255,255,0) #FFFFCC rgba(255,255,255,0) rgba(255,255,255,0)"); + // The original left position of the tooltip_notification's arrow border is -16px + tooltip_notification.children("div[class='ttn_arrow_border']").css("left", "").css("left", "-16px"); + tooltip_notification.children("div[class='ttn_arrow_border']").css("border-color", "transparent #FFCC00 transparent transparent"); + tooltip_notification.children("div[class='ttn_arrow_border']").css("border-color", "rgba(255,255,255,0) #FFCC00 rgba(255,255,255,0) rgba(255,255,255,0)"); + tooltip_notification.css("-moz-box-shadow", "1px 1px 3px gray"); + tooltip_notification.css("-webkit-box-shadow", "1px 1px 3px gray"); + tooltip_notification.css("-o-box-shadow", "1px 1px 3px gray"); + tooltip_notification.css("box-shadow", "1px 1px 3px gray"); + } + + // Set the final attributes for tooltip_notification + tooltip_notification.css('top', tooltip_notification_top + 'px'); + tooltip_notification.css('left', tooltip_notification_left + 'px'); + + // If the tooltip_notification was previously hidden, show it. + if ( !tooltip_notification.is(":visible") ) { + tooltip_notification.show(); + } + + } + + else { + + // If the tooltip_notification was previously visible, hide it. + if ( tooltip_notification.is(":visible") ) { + tooltip_notification.hide(); + } + + } + +} diff --git a/modules/webnews/lib/webnews.py b/modules/webnews/lib/webnews.py new file mode 100644 index 0000000000..a1d15dfae8 --- /dev/null +++ b/modules/webnews/lib/webnews.py @@ -0,0 +1,291 @@ +# -*- coding: utf-8 -*- +## +## This file is part of Invenio. +## Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +""" WebNews module """ + +__revision__ = "$Id$" + +# GENERAL IMPORTS +from cgi import escape +import sys +CFG_JSON_AVAILABLE = True +if sys.hexversion < 0x2060000: + try: + import simplejson as json + except: + CFG_JSON_AVAILABLE = False +else: + import json + +# GENERAL IMPORTS +from urlparse import urlsplit +import time + +# INVENIO IMPORTS +from invenio.config import CFG_SITE_LANG +from invenio.webinterface_handler_wsgi_utils import Cookie, \ + get_cookie + # INFO: Old API, ignore + #add_cookies, \ +from invenio.messages import gettext_set_language +from invenio.urlutils import get_referer + +# MODULE IMPORTS +from invenio.webnews_dblayer import get_latest_story_id, \ + get_story_tooltips +from invenio.webnews_config import CFG_WEBNEWS_TOOLTIPS_DISPLAY, \ + CFG_WEBNEWS_TOOLTIPS_COOKIE_LONGEVITY, \ + CFG_WEBNEWS_TOOLTIPS_COOKIE_NAME + +def _create_tooltip_cookie(name = CFG_WEBNEWS_TOOLTIPS_COOKIE_NAME, + value = "", + path = "/", + longevity = CFG_WEBNEWS_TOOLTIPS_COOKIE_LONGEVITY): + """ + Private shortcut function that returns an instance of a Cookie for the + tooltips. + """ + + # The local has to be English for this to work! + longevity_time = time.time() + ( longevity * 24 * 60 * 60 ) + longevity_expression = time.strftime("%a, %d-%b-%Y %T GMT", time.gmtime(longevity_time)) + + cookie = Cookie(name, + value, + path = path, + expires = longevity_expression) + + return cookie + +def _paths_match(path1, path2): + """ + Internal path matcher. + "*" acts as a wildcard for individual path parts. + """ + + # Start off by assuming that the paths don't match + paths_match_p = False + + # Get the individual path parts. + path1_parts = path1.strip("/").split("/") + path2_parts = path2.strip("/").split("/") + + # If the 2 paths have different number of parts don't even bother checking + if len(path1_parts) == len(path2_parts): + # Check if the individual path parts match + for (part1, part2) in zip(path1_parts, path2_parts): + paths_match_p = ( part1 == part2 ) or ( "*" in (part1, part2) ) + if not paths_match_p: + break + + return paths_match_p + +def _does_referer_match_target_page(referer, + target_page): + """ + Compares the referer and the target page in a smart way and + returns True if they match, otherwise False. + """ + + try: + return ( target_page == "*" ) or _paths_match(urlsplit(target_page)[2], urlsplit(referer)[2]) + except: + return False + +def perform_request_tooltips(req = None, + uid = 0, + story_id = 0, + tooltip_id = 0, + ln = CFG_SITE_LANG): + """ + Calculates and returns the tooltips information in JSON. + """ + + tooltips_dict = {} + + #tooltips_json = json.dumps(tooltips_dict) + tooltips_json = '{}' + + # Did we import json? + # Should we display tooltips at all? + # Does the request exist? + if not CFG_JSON_AVAILABLE or not CFG_WEBNEWS_TOOLTIPS_DISPLAY or req is None: + return tooltips_json + + if story_id == 0: + # Is there a latest story to display? + story_id = get_latest_story_id() + + if story_id is None: + return tooltips_json + else: + # Are there any tooltips associated to this story? + # TODO: Filter the unwanted tooltips in the DB query. + # We can already filter by REFERER and by the tooltips IDs in the cookie + # In that case we don't have to iterate through the tooltips later and + # figure out which ones to keep. + tooltips = get_story_tooltips(story_id) + if tooltips is None: + return tooltips_json + + # In a more advance scenario we would save the information on whether the + # the user has seen the tooltips: + # * in a session param for the users that have logged in + # * in a cookie for guests + # We could then use a combination of these two to decide whether to display + # the tooltips or not. + # + # In that case, the following tools could be used: + #from invenio.webuser import isGuestUser, \ + # session_param_get, \ + # session_param_set + #is_user_guest = isGuestUser(uid) + #if not is_user_guest: + # try: + # tooltip_information = session_param_get(req, CFG_WEBNEWS_TOOLTIPS_SESSION_PARAM_NAME) + # except KeyError: + # session_param_set(req, CFG_WEBNEWS_TOOLTIPS_SESSION_PARAM_NAME, "") + + cookie_name = "%s_%s" % (CFG_WEBNEWS_TOOLTIPS_COOKIE_NAME, str(story_id)) + + try: + # Get the cookie + cookie = get_cookie(req, cookie_name) + # Get the tooltip IDs that have already been displayed + tooltips_in_cookie = filter(None, str(cookie.value).split(",")) + except: + # TODO: Maybe set a cookie with an emptry string as value? + tooltips_in_cookie = [] + + # Prepare the user's prefered language and labels. + _ = gettext_set_language(ln) + readmore_label = _("Learn more") + dismiss_label = _("I got it!") + + # Get the referer, in order to check if we should display + # the tooltip in the given page. + referer = get_referer(req) + + tooltips_list = [] + + for tooltip in tooltips: + tooltip_notification_id = 'ttn_%s_%s' % (str(story_id), str(tooltip[0])) + # INFO: the tooltip body is not escaped! + # it's up to the admin to insert proper body text. + #tooltip_body = escape(tooltip[1], True) + tooltip_body = tooltip[1] + tooltip_target_element = tooltip[2] + tooltip_target_page = tooltip[3] + + # Only display the tooltips that match the referer and that the user + # has not already seen. + if _does_referer_match_target_page(referer, tooltip_target_page) and \ + ( tooltip_notification_id not in tooltips_in_cookie ): + + # Add this tooltip to the tooltips that we will display. + tooltips_list.append({ + 'id' : tooltip_notification_id, + 'target' : tooltip_target_element, + 'body' : tooltip_body, + 'readmore' : readmore_label, + 'dismiss' : dismiss_label, + }) + + # Add this tooltip to the tooltips that the user has already seen. + #tooltips_in_cookie.append(tooltip_notification_id) + + if tooltips_list: + # Hooray! There are some tooltips to display! + tooltips_dict['tooltips'] = tooltips_list + tooltips_dict['story_id'] = str(story_id) + tooltips_dict['ln'] = ln + + # Create and set the updated cookie. + #cookie_value = ",".join(tooltips_in_cookie) + #cookie = _create_tooltip_cookie(cookie_name, + # cookie_value) + #req.set_cookie(cookie) + ## INFO: Old API, ignore + ##add_cookies(req, [cookie]) + + # JSON-ify and return the tooltips. + tooltips_json = json.dumps(tooltips_dict) + return tooltips_json + +def perform_request_dismiss(req = None, + uid = 0, + story_id = 0, + tooltip_notification_id = None): + """ + Dismisses the given tooltip for the current user. + """ + + try: + + if not CFG_JSON_AVAILABLE or not CFG_WEBNEWS_TOOLTIPS_DISPLAY or req is None: + raise Exception("Tooltips are not currently available.") + + # Retrieve the story_id + if story_id == 0: + if tooltip_notification_id is None: + raise Exception("No tooltip_notification_id has been given.") + else: + story_id = tooltip_notification_id.split("_")[1] + + # Generate the cookie name out of the story_id + cookie_name = "%s_%s" % (CFG_WEBNEWS_TOOLTIPS_COOKIE_NAME, str(story_id)) + + # Get the existing tooltip_notification_ids from the cookie + try: + # Get the cookie + cookie = get_cookie(req, cookie_name) + # Get the tooltip IDs that have already been displayed + tooltips_in_cookie = filter(None, str(cookie.value).split(",")) + except: + # TODO: Maybe set a cookie with an emptry string as value? + tooltips_in_cookie = [] + + # Append the tooltip_notification_id to the existing tooltip_notification_ids + # (only if it's not there already ; but normally it shouldn't be) + if tooltip_notification_id not in tooltips_in_cookie: + tooltips_in_cookie.append(tooltip_notification_id) + + # Create and set the cookie with the updated cookie value + cookie_value = ",".join(tooltips_in_cookie) + cookie = _create_tooltip_cookie(cookie_name, cookie_value) + req.set_cookie(cookie) + # INFO: Old API, ignore + #add_cookies(req, [cookie]) + + except: + # Something went wrong.. + # TODO: what went wrong? + dismissed_p_dict = { "success" : 0 } + dismissed_p_json = json.dumps(dismissed_p_dict) + return dismissed_p_json + else: + # Everything went great! + dismissed_p_dict = { "success" : 1 } + dismissed_p_json = json.dumps(dismissed_p_dict) + return dismissed_p_json + # enable for python >= 2.5 + #finally: + # # JSON-ify and return the result + # dismissed_p_json = json.dumps(dismissed_p_dict) + # return dismissed_p_json diff --git a/modules/webnews/lib/webnews_config.py b/modules/webnews/lib/webnews_config.py new file mode 100644 index 0000000000..c3154176f0 --- /dev/null +++ b/modules/webnews/lib/webnews_config.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +## +## This file is part of Invenio. +## Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +""" WebNews module configuration """ + +__revision__ = "$Id$" + +# Should we generally display tooltips or not? +CFG_WEBNEWS_TOOLTIPS_DISPLAY = True + +# The tooltips session param name +#CFG_WEBNEWS_TOOLTIPS_SESSION_PARAM_NAME = "has_user_seen_tooltips" + +# Tooltips cookie settings +# The cookie name +CFG_WEBNEWS_TOOLTIPS_COOKIE_NAME = "INVENIOTOOLTIPS" +# the cookie longevity in days +CFG_WEBNEWS_TOOLTIPS_COOKIE_LONGEVITY = 14 diff --git a/modules/webnews/lib/webnews_dblayer.py b/modules/webnews/lib/webnews_dblayer.py new file mode 100644 index 0000000000..024abfbc1a --- /dev/null +++ b/modules/webnews/lib/webnews_dblayer.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +## +## This file is part of Invenio. +## Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +""" Database related functions for the WebNews module """ + +__revision__ = "$Id$" + +# INVENIO IMPORTS +from invenio.dbquery import run_sql + +# MODULE IMPORTS +from invenio.webnews_utils import convert_xpath_expression_to_jquery_selector +from invenio.webnews_config import CFG_WEBNEWS_TOOLTIPS_COOKIE_LONGEVITY + +def get_latest_story_id(): + """ + Returns the id of the latest news story available. + """ + + query = """ SELECT id + FROM nwsSTORY + WHERE created >= DATE_SUB(CURDATE(),INTERVAL %s DAY) + ORDER BY created DESC + LIMIT 1""" + + params = (CFG_WEBNEWS_TOOLTIPS_COOKIE_LONGEVITY,) + + res = run_sql(query, params) + + if res: + return res[0][0] + + return None + +def get_story_tooltips(story_id): + """ + Returns all the available tooltips for the given story ID. + """ + + query = """ SELECT id, + body, + target_element, + target_page + FROM nwsTOOLTIP + WHERE id_story=%s""" + + params = (story_id,) + + res = run_sql(query, params) + + if res: + return res + return None + +def update_tooltip(story_id, + tooltip_id, + tooltip_body, + tooltip_target_element, + tooltip_target_page, + is_tooltip_target_xpath = False): + """ + Updates the tooltip information. + XPath expressions are automatically translated to the equivalent jQuery + selector if so chosen by the user. + """ + + query = """ UPDATE nwsTOOLTIP + SET body=%s, + target_element=%s, + target_page=%s + WHERE id=%s + AND id_story=%s""" + + tooltip_target_element = is_tooltip_target_xpath and \ + convert_xpath_expression_to_jquery_selector(tooltip_target_element) or \ + tooltip_target_element + + params = (tooltip_body, tooltip_target_element, tooltip_target_page, tooltip_id, story_id) + + res = run_sql(query, params) + + return res diff --git a/modules/webnews/lib/webnews_utils.py b/modules/webnews/lib/webnews_utils.py new file mode 100644 index 0000000000..62915fa271 --- /dev/null +++ b/modules/webnews/lib/webnews_utils.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +## +## This file is part of Invenio. +## Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +""" Utlis for the WebNews module """ + +__revision__ = "$Id$" + +# GENERAL IMPORTS +import re + +# CONSTANT VARIABLES +# Regex for the nth element +PAT_NTH = r'(\w+)\[(\d+)\]' +def REP_NTH(matchobj): + return "%s:eq(%s)" % (str(matchobj.group(1)), str(int(matchobj.group(2))-1)) +FLG_NTH = 0 +# Regex for the id attribute +PAT_IDA = r'\*\[@id=[\'"]([\w\-]+)[\'"]\]' +REP_IDA = r'#\1' +FLG_IDA = 0 + +def convert_xpath_expression_to_jquery_selector(xpath_expression): + """ + Given an XPath expression this function + returns the equivalent jQuery selector. + """ + + tmp_result = xpath_expression.strip('/') + tmp_result = tmp_result.split('/') + tmp_result = [_x2j(e) for e in zip(tmp_result, range(len(tmp_result)))] + jquery_selector = '.'.join(tmp_result) + + return jquery_selector + +def _x2j((s, i)): + """ + Private helper function that converts each element of an XPath expression + to the equivalent jQuery selector using regular expressions. + """ + + s = re.sub(PAT_IDA, REP_IDA, s, FLG_IDA) + s = re.sub(PAT_NTH, REP_NTH, s, FLG_NTH) + s = '%s("%s")' % (i and 'children' or '$', s) + + return s diff --git a/modules/webnews/lib/webnews_webinterface.py b/modules/webnews/lib/webnews_webinterface.py new file mode 100644 index 0000000000..2af61fedc8 --- /dev/null +++ b/modules/webnews/lib/webnews_webinterface.py @@ -0,0 +1,81 @@ +## This file is part of Invenio. +## Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +"""WebNews Web Interface.""" + +__revision__ = "$Id$" + +__lastupdated__ = """$Date$""" + +# INVENIO IMPORTS +from invenio.webinterface_handler import wash_urlargd, WebInterfaceDirectory +from invenio.config import CFG_ACCESS_CONTROL_LEVEL_SITE, \ + CFG_SITE_LANG +from invenio.webuser import getUid, page_not_authorized + +# MODULE IMPORTS +from invenio.webnews import perform_request_tooltips, \ + perform_request_dismiss + +class WebInterfaceWebNewsPages(WebInterfaceDirectory): + """ + Defines the set of /news pages. + """ + + _exports = ["tooltips", "dismiss"] + + def tooltips(self, req, form): + """ + Returns the news tooltips information in JSON. + """ + + argd = wash_urlargd(form, {'story_id' : (int, 0), + 'tooltip_id' : (int, 0), + 'ln' : (str, CFG_SITE_LANG)}) + + uid = getUid(req) + if uid == -1 or CFG_ACCESS_CONTROL_LEVEL_SITE >= 1: + return page_not_authorized(req, "../news/tooltip", + navmenuid = 'news') + + tooltips_json = perform_request_tooltips(req = req, + uid=uid, + story_id=argd['story_id'], + tooltip_id=argd['tooltip_id'], + ln=argd['ln']) + + return tooltips_json + + def dismiss(self, req, form): + """ + Dismiss the given tooltip for the current user. + """ + + argd = wash_urlargd(form, {'story_id' : (int, 0), + 'tooltip_notification_id' : (str, None)}) + + uid = getUid(req) + if uid == -1 or CFG_ACCESS_CONTROL_LEVEL_SITE >= 1: + return page_not_authorized(req, "../news/dismiss", + navmenuid = 'news') + + dismissed_p_json = perform_request_dismiss(req = req, + uid=uid, + story_id=argd['story_id'], + tooltip_notification_id=argd['tooltip_notification_id']) + + return dismissed_p_json diff --git a/modules/webnews/web/Makefile.am b/modules/webnews/web/Makefile.am new file mode 100644 index 0000000000..a6a1d24e56 --- /dev/null +++ b/modules/webnews/web/Makefile.am @@ -0,0 +1,18 @@ +## This file is part of Invenio. +## Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +CLEANFILES = *~ *.tmp diff --git a/modules/webstyle/lib/webinterface_layout.py b/modules/webstyle/lib/webinterface_layout.py index 494dbed8d6..7534523114 100644 --- a/modules/webstyle/lib/webinterface_layout.py +++ b/modules/webstyle/lib/webinterface_layout.py @@ -301,6 +301,12 @@ def _lookup(self, component, path): register_exception(alert_admin=True, subject='EMERGENCY') WebInterfaceAuthorlistPages = WebInterfaceDumbPages +try: + from invenio.webnews_webinterface import WebInterfaceWebNewsPages +except: + register_exception(alert_admin=True, subject='EMERGENCY') + WebInterfaceWebNewsPages = WebInterfaceDumbPages + if CFG_OPENAIRE_SITE: try: from invenio.openaire_deposit_webinterface import \ @@ -369,6 +375,7 @@ class WebInterfaceInvenio(WebInterfaceSearchInterfacePages): 'goto', 'info', 'authorlist', + 'news', ] + test_exports + openaire_exports def __init__(self): @@ -410,6 +417,7 @@ def __init__(self): yourcomments = WebInterfaceDisabledPages() goto = WebInterfaceDisabledPages() authorlist = WebInterfaceDisabledPages() + news = WebInterfaceDisabledPages() else: submit = WebInterfaceSubmitPages() youraccount = WebInterfaceYourAccountPages() @@ -442,7 +450,7 @@ def __init__(self): yourcomments = WebInterfaceYourCommentsPages() goto = WebInterfaceGotoPages() authorlist = WebInterfaceAuthorlistPages() - + news = WebInterfaceWebNewsPages() # This creates the 'handler' function, which will be invoked directly # by mod_python. diff --git a/modules/webstyle/lib/webstyle_templates.py b/modules/webstyle/lib/webstyle_templates.py index 316620ebaa..435de02d68 100644 --- a/modules/webstyle/lib/webstyle_templates.py +++ b/modules/webstyle/lib/webstyle_templates.py @@ -42,6 +42,8 @@ CFG_INSPIRE_SITE, \ CFG_WEBLINKBACK_TRACKBACK_ENABLED +from invenio.webnews_config import CFG_WEBNEWS_TOOLTIPS_DISPLAY + from invenio.messages import gettext_set_language, language_list_long, is_language_rtl from invenio.urlutils import make_canonical_urlargd, create_html_link, \ get_canonical_and_alternates_urls @@ -389,6 +391,10 @@ def tmpl_pageheader(self, req, ln=CFG_SITE_LANG, headertitle="", %(hepDataAdditions)s + + + + %(metaheaderadd)s @@ -546,6 +552,25 @@ def tmpl_pagefooter(self, req=None, ln=CFG_SITE_LANG, lastupdated=None, else: msg_lastupdated = "" + if CFG_WEBNEWS_TOOLTIPS_DISPLAY: + tooltips_script = """ + +""" + else: + tooltips_script = "" + out = """
    %(pagefooteradd)s @@ -564,6 +589,7 @@ def tmpl_pagefooter(self, req=None, ln=CFG_SITE_LANG, lastupdated=None,
    +%(tooltips_script)s """ % { @@ -588,6 +614,8 @@ def tmpl_pagefooter(self, req=None, ln=CFG_SITE_LANG, lastupdated=None, 'version': CFG_VERSION, 'pagefooteradd': pagefooteradd, + + 'tooltips_script' : tooltips_script, } return out From c20a6cb567b1bc436acabc61dc40b18fbb4fda36 Mon Sep 17 00:00:00 2001 From: "Esteban J. G. Gabancho" Date: Thu, 15 Oct 2015 16:05:33 +0200 Subject: [PATCH 44/59] WebSearch: guess primary collection for CERN Site --- modules/websearch/lib/search_engine.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/websearch/lib/search_engine.py b/modules/websearch/lib/search_engine.py index 73a77522a7..7c76a71c3f 100644 --- a/modules/websearch/lib/search_engine.py +++ b/modules/websearch/lib/search_engine.py @@ -3689,6 +3689,11 @@ def guess_primary_collection_of_a_record(recID): if recID in get_collection_reclist(coll_name): return coll_name + # even more dirty for CERN Website Archive + for coll in dbcollids: + if coll == 'WEBPAGE' and recID in get_collection_reclist('CERN Website'): + return 'CERN Website' + return out _re_collection_url = re.compile('/collection/(.+)') From c09cdce9c2ca18a7564909d217d56d8352ea1f57 Mon Sep 17 00:00:00 2001 From: Harris Tzovanakis Date: Thu, 4 Dec 2014 15:21:09 +0100 Subject: [PATCH 45/59] WebStat: register custom events on es * Adds `loanrequest` schema for `Lumberjack`. * Adds custom events elastic search logging. Signed-off-by: Harris Tzovanakis --- modules/webstat/lib/webstat.py | 48 ++++++++++++++++++++++----- modules/webstat/lib/webstat_config.py | 26 ++++++++++++++- 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/modules/webstat/lib/webstat.py b/modules/webstat/lib/webstat.py index f5e440eea6..7232fc052e 100644 --- a/modules/webstat/lib/webstat.py +++ b/modules/webstat/lib/webstat.py @@ -26,6 +26,7 @@ import calendar from datetime import timedelta from urllib import quote +from logging import getLogger from invenio import template from invenio.config import \ @@ -33,8 +34,11 @@ CFG_TMPDIR, \ CFG_SITE_URL, \ CFG_SITE_LANG, \ - CFG_WEBSTAT_BIBCIRCULATION_START_YEAR -from invenio.webstat_config import CFG_WEBSTAT_CONFIG_PATH + CFG_WEBSTAT_BIBCIRCULATION_START_YEAR, \ + CFG_ELASTICSEARCH_LOGGING +from invenio.webstat_config import \ + CFG_WEBSTAT_CONFIG_PATH, \ + CFG_ELASTICSEARCH_EVENTS_MAP from invenio.bibindex_engine_utils import get_all_indexes from invenio.bibindex_tokenizers.BibIndexJournalTokenizer import CFG_JOURNAL_TAG from invenio.search_engine import get_coll_i18nname, \ @@ -724,6 +728,7 @@ def destroy_customevents(): msg += destroy_customevent(event[0]) return msg + def register_customevent(event_id, *arguments): """ Registers a custom event. Will add to the database's event tables @@ -739,17 +744,25 @@ def register_customevent(event_id, *arguments): @param *arguments: The rest of the parameters of the function call @type *arguments: [params] """ - res = run_sql("SELECT CONCAT('staEVENT', number),cols " + \ - "FROM staEVENT WHERE id = %s", (event_id, )) + query = """ + SELECT CONCAT('staEVENT', number), + cols + FROM staEVENT + WHERE id = %s + """ + params = (event_id,) + res = run_sql(query, params) + # the id does not exist if not res: - return # the id does not exist + return tbl_name = res[0][0] if res[0][1]: col_titles = cPickle.loads(res[0][1]) else: col_titles = [] if len(col_titles) != len(arguments[0]): - return # there is different number of arguments than cols + # there is different number of arguments than cols + return # Make sql query if len(arguments[0]) != 0: @@ -758,18 +771,35 @@ def register_customevent(event_id, *arguments): for title in col_titles: sql_query.append("`%s`" % title) sql_query.append(",") - sql_query.pop() # del the last ',' + # del the last ',' + sql_query.pop() sql_query.append(") VALUES (") for argument in arguments[0]: sql_query.append("%s") sql_query.append(",") sql_param.append(argument) - sql_query.pop() # del the last ',' + # del the last ',' + sql_query.pop() sql_query.append(")") sql_str = ''.join(sql_query) run_sql(sql_str, tuple(sql_param)) + + # Register the event on elastic search + if CFG_ELASTICSEARCH_LOGGING and event_id in \ + CFG_ELASTICSEARCH_EVENTS_MAP.keys(): + # Initialize elastic search handler + elastic_search_parameters = zip(col_titles, arguments[0]) + event_logger_name = "events.{0}".format(event_id) + logger = getLogger(event_logger_name) + log_event = {} + for key, value in elastic_search_parameters: + log_event[key] = value + logger.info(log_event) else: - run_sql("INSERT INTO %s () VALUES ()" % wash_table_column_name(tbl_name)) # kwalitee: disable=sql + # kwalitee: disable=sql + run_sql( + "INSERT INTO %s () VALUES ()" % wash_table_column_name(tbl_name) + ) def cache_keyevent_trend(ids=[]): diff --git a/modules/webstat/lib/webstat_config.py b/modules/webstat/lib/webstat_config.py index 917dd71aac..da93dc732d 100644 --- a/modules/webstat/lib/webstat_config.py +++ b/modules/webstat/lib/webstat_config.py @@ -21,6 +21,30 @@ __revision__ = "$Id$" -from invenio.config import CFG_ETCDIR +from invenio.config import CFG_ETCDIR, CFG_ELASTICSEARCH_LOGGING, CFG_CERN_SITE CFG_WEBSTAT_CONFIG_PATH = CFG_ETCDIR + "/webstat/webstat.cfg" + +if CFG_CERN_SITE: + CFG_ELASTICSEARCH_EVENTS_MAP = { + "events.loanrequest": { + "_source": { + "enabled": True + }, + "properties": { + "request_id": { + "type": "integer" + }, + "load_id": { + "type": "integer" + } + } + } + } +else: + CFG_ELASTICSEARCH_EVENTS_MAP = {} + +if CFG_ELASTICSEARCH_LOGGING: + from invenio.elasticsearch_logging import register_schema + for name, arguments in CFG_ELASTICSEARCH_EVENTS_MAP.items(): + register_schema(name, arguments) From 4accd0709a503cb1cc4945a592cb4a2a046ff289 Mon Sep 17 00:00:00 2001 From: Joe MacMahon Date: Mon, 3 Aug 2015 17:48:21 +0200 Subject: [PATCH 46/59] elasticsearch: GeoIP in Elasticsearch logging * NEW Adds a new ip_field column to staEVENT * Uses this column to run GeoIP lookup on logged events Signed-off-by: Joe MacMahon --- modules/miscutil/sql/tabcreate.sql | 1 + modules/webstat/lib/webstat.py | 54 +++++++++++++++++++++++------ modules/webstat/lib/webstatadmin.py | 7 ++-- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/modules/miscutil/sql/tabcreate.sql b/modules/miscutil/sql/tabcreate.sql index 648f3fe4e2..af36f60830 100644 --- a/modules/miscutil/sql/tabcreate.sql +++ b/modules/miscutil/sql/tabcreate.sql @@ -4279,6 +4279,7 @@ CREATE TABLE IF NOT EXISTS staEVENT ( name varchar(255), creation_time TIMESTAMP DEFAULT NOW(), cols varchar(255), + ip_field varchar(255), PRIMARY KEY (id), UNIQUE KEY number (number) ) ENGINE=MyISAM; diff --git a/modules/webstat/lib/webstat.py b/modules/webstat/lib/webstat.py index 7232fc052e..de7b771346 100644 --- a/modules/webstat/lib/webstat.py +++ b/modules/webstat/lib/webstat.py @@ -27,6 +27,7 @@ from datetime import timedelta from urllib import quote from logging import getLogger +from lumberjack.postprocessors import geoip from invenio import template from invenio.config import \ @@ -515,7 +516,7 @@ def get_collection_list_plus_all(): # CLI -def create_customevent(event_id=None, name=None, cols=[]): +def create_customevent(event_id=None, name=None, cols=[], ip_field=None): """ Creates a new custom event by setting up the necessary MySQL tables. @@ -528,6 +529,9 @@ def create_customevent(event_id=None, name=None, cols=[]): @param cols: Optionally, the name of the additional columns. @type cols: [str] + @param ip_field: Optionally, the name of a field containing an IP + @type ip_field: str + @return: A status message @type: str """ @@ -549,6 +553,10 @@ def create_customevent(event_id=None, name=None, cols=[]): if (argument == "creation_time") or (argument == "id"): return "Invalid column title: %s! Aborted." % argument + # Check the IP field is in cols + if ip_field is not None and ip_field not in cols: + return "Specified ID field '%s' not in column names provided! Aborted." % ip_field + # Insert a new row into the events table describing the new event sql_param = [event_id] if name is not None: @@ -561,8 +569,14 @@ def create_customevent(event_id=None, name=None, cols=[]): sql_param.append(cPickle.dumps(cols)) else: sql_cols = "NULL" - run_sql("INSERT INTO staEVENT (id, name, cols) VALUES (%s, " + \ - sql_name + ", " + sql_cols + ")", tuple(sql_param)) + if ip_field is not None: + sql_ip_field = "%s" + sql_param.append(ip_field) + else: + sql_ip_field = "NULL" + run_sql("INSERT INTO staEVENT (id, name, cols, ip_field) VALUES (%s, " + \ + sql_name + ", " + sql_cols + ", " + sql_ip_field + ")", + tuple(sql_param)) tbl_name = get_customevent_table(event_id) @@ -584,7 +598,7 @@ def create_customevent(event_id=None, name=None, cols=[]): % (tbl_name, event_id) -def modify_customevent(event_id=None, name=None, cols=[]): +def modify_customevent(event_id=None, name=None, cols=[], ip_field=None): """ Modify a custom event. It can modify the columns definition or/and the descriptive name @@ -598,6 +612,9 @@ def modify_customevent(event_id=None, name=None, cols=[]): @param cols: Optionally, the name of the additional columns. @type cols: [str] + @param ip_field: Optionally, the name of a field containing an IP + @type ip_field: str + @return: A status message @type: str """ @@ -618,10 +635,14 @@ def modify_customevent(event_id=None, name=None, cols=[]): "FROM staEVENT WHERE id = %s", (event_id, )) if not res: return "Invalid event id: %s! Aborted" % event_id - if not run_sql("SHOW TABLES LIKE %s", res[0][0]): - run_sql("DELETE FROM staEVENT WHERE id=%s", (event_id, )) - create_customevent(event_id, event_id, cols) - return + try: + if not run_sql("SHOW TABLES LIKE %s", res[0][0]): + run_sql("DELETE FROM staEVENT WHERE id=%s", (event_id, )) + create_customevent(event_id, event_id, cols) + return + except TypeError as e: + print(repr(res[0])) + raise cols_orig = cPickle.loads(res[0][1]) # add new cols @@ -676,6 +697,13 @@ def modify_customevent(event_id=None, name=None, cols=[]): sql_query.append("name = %s") sql_query.append(",") sql_param.append(name) + if ip_field is not None: + sql_query.append("ip_field = %s") + sql_query.append(",") + sql_param.append(ip_field) + else: + sql_query.append("ip_field = NULL") + sql_query.append(",") if sql_param: sql_query[-1] = "WHERE id = %s" sql_param.append(event_id) @@ -746,7 +774,8 @@ def register_customevent(event_id, *arguments): """ query = """ SELECT CONCAT('staEVENT', number), - cols + cols, + ip_field FROM staEVENT WHERE id = %s """ @@ -763,6 +792,7 @@ def register_customevent(event_id, *arguments): if len(col_titles) != len(arguments[0]): # there is different number of arguments than cols return + ip_field = res[0][2] # Make sql query if len(arguments[0]) != 0: @@ -794,7 +824,11 @@ def register_customevent(event_id, *arguments): log_event = {} for key, value in elastic_search_parameters: log_event[key] = value - logger.info(log_event) + if ip_field is not None: + postprocessors = [geoip(field=ip_field)] + else: + postprocessors = [] + logger.info(log_event, {'postprocessors': postprocessors}) else: # kwalitee: disable=sql run_sql( diff --git a/modules/webstat/lib/webstatadmin.py b/modules/webstat/lib/webstatadmin.py index 0df038b9f8..362a36de8e 100644 --- a/modules/webstat/lib/webstatadmin.py +++ b/modules/webstat/lib/webstatadmin.py @@ -194,6 +194,7 @@ def task_submit_check_options(): if section[:21] == "webstat_custom_event_": cols = [] name = "" + ip_field = None for option, value in conf.items(section): if option == "name": name = value @@ -203,14 +204,16 @@ def task_submit_check_options(): while len(cols) <= index: cols.append("") cols[index] = value + if option == "ip_field": + ip_field = value if name: res = run_sql("SELECT COUNT(id) FROM staEVENT WHERE id = %s", (name, )) if res[0][0] == 0: # name does not exist, create customevent - webstat.create_customevent(name, name, cols) + webstat.create_customevent(name, name, cols, ip_field) else: # name already exists, update customevent - webstat.modify_customevent(name, cols=cols) + webstat.modify_customevent(name, cols=cols, ip_field=ip_field) sys.exit(0) From 7798090bf438e2f02422c026aeebc6f3c8f4ac7c Mon Sep 17 00:00:00 2001 From: Harris Tzovanakis Date: Fri, 4 Sep 2015 09:32:41 +0200 Subject: [PATCH 47/59] WebSubmit: fix authors autocomplete * FIX Attaches `typeahead` autocomplete plugin to more specific element. Signed-off-by: Harris Tzovanakis --- modules/websubmit/lib/websubmit_templates.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/websubmit/lib/websubmit_templates.py b/modules/websubmit/lib/websubmit_templates.py index fec9602e68..7db42d3bcf 100644 --- a/modules/websubmit/lib/websubmit_templates.py +++ b/modules/websubmit/lib/websubmit_templates.py @@ -34,6 +34,7 @@ from invenio.webmessage_mailutils import email_quoted_txt2html from invenio.htmlutils import escape_html, escape_javascript_string from invenio.websubmit_config import CFG_WEBSUBMIT_CHECK_USER_LEAVES_SUBMISSION +from invenio.websubmit_functions.ParamFile import ParamFromFile class Template: @@ -3110,7 +3111,7 @@ def tmpl_authors_autocompletion( 'email': (datum['email'] !== undefined) ? datum['email'] : '' }); } - $('.typeahead').typeahead('val', ''); + $('#author_textbox').typeahead('val', ''); } function appendRow(author) { @@ -3258,7 +3259,7 @@ def tmpl_authors_autocompletion( engine.initialize(); } - $('.typeahead').typeahead({ + $('#author_textbox').typeahead({ highlight: true, hint: true, minLength: 3 From 9725cd4cad1aca7ea5af7703d75245deb7e968e6 Mon Sep 17 00:00:00 2001 From: Harris Tzovanakis Date: Mon, 7 Sep 2015 17:35:09 +0200 Subject: [PATCH 48/59] oaiharvest: authorlist missing * FIX Checks only for error code in bibconvert of authorlist, because stdoutput warnings caused autholist to be ignored. Signed-off-by: Harris Tzovanakis --- modules/oaiharvest/lib/oai_harvest_daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/oaiharvest/lib/oai_harvest_daemon.py b/modules/oaiharvest/lib/oai_harvest_daemon.py index 546b235b6d..b87e241f40 100644 --- a/modules/oaiharvest/lib/oai_harvest_daemon.py +++ b/modules/oaiharvest/lib/oai_harvest_daemon.py @@ -1156,7 +1156,7 @@ def authorlist_extract(tarball_path, identifier, downloaded_files, stylesheet): exitcode, cmd_stderr = call_bibconvert(config=stylesheet, \ harvestpath=temp_authorlist_path, \ convertpath=authorlist_resultxml_path) - if cmd_stderr == "" and exitcode == 0: + if exitcode == 0: # Success! return 0, "", authorlist_resultxml_path # No valid authorlist found From 97891ff3457c02b601311b54b2275a3fb394b96c Mon Sep 17 00:00:00 2001 From: "Esteban J. G. Gabancho" Date: Fri, 20 Nov 2015 16:47:52 +0100 Subject: [PATCH 49/59] WebSubmit: report number generation enhacement * Adds a new predefined keyword for year generation. Co-Authored-By: Ludmila Marian Signed-off-by: Ludmila Marian --- .../lib/functions/Report_Number_Generation.py | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/modules/websubmit/lib/functions/Report_Number_Generation.py b/modules/websubmit/lib/functions/Report_Number_Generation.py index 2ab0652466..a16230149a 100644 --- a/modules/websubmit/lib/functions/Report_Number_Generation.py +++ b/modules/websubmit/lib/functions/Report_Number_Generation.py @@ -1,5 +1,5 @@ # This file is part of Invenio. -# Copyright (C) 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 CERN. +# Copyright (C) 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 CERN. # # Invenio is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -14,10 +14,12 @@ # You should have received a copy of the GNU General Public License # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + """Description: function Report_Number_Generation This function creates a reference for the submitted document and saves it in the specified file. Author: T.Baron""" + __revision__ = "$Id$" import os @@ -26,6 +28,7 @@ import fcntl import errno +from invenio.search_engine import get_creation_date, get_fieldvalues from invenio.config import CFG_WEBSUBMIT_COUNTERSDIR from invenio.websubmit_config import InvenioWebSubmitFunctionError from invenio.shellutils import mymkdir @@ -69,8 +72,10 @@ def Report_Number_Generation(parameters, curdir, form, user_info=None): a file genereated during submission, matching [re] separated by - (dash) char . - * yeargen: if "AUTO", current year, else the year is - extracted from the file [yeargen] + * yeargen: year generation. You can use: + "AUTO", for current year + "CREATION", for year of document (publication, if not found, record creation) + else, extracted from the file [yeargen] * nblength: the number of digits for the report number. Eg: '3' for XXX-YYYY-025 or '4' @@ -95,7 +100,24 @@ def Report_Number_Generation(parameters, curdir, form, user_info=None): if parameters['yeargen'] == "AUTO": # Current year is used yy = time.strftime("%Y") - else : + elif parameters['yeargen'] == 'CREATION': + # Publication date (only if the record exits) + fp = open("%s/%s" % (curdir, 'SN'), "r") + recid = fp.read() + fp.close() + yy = '' + try: + # first see if the publication date is available + yy = get_fieldvalues(recid, '260__c')[0] + if not yy: + # if the publication date is not available, check the imprint date + yy = get_fieldvalues(recid, '269__c')[0][-4:] + except IndexError: + pass + if not re.match('\d{4}', yy): + # fallback on the record creation date + yy = get_creation_date(recid, '%Y') + else: # If yeargen != auto then the contents of the file named 'yeargen' is used # Assumes file uses DD-MM-YYYY format fp = open("%s/%s" % (curdir, parameters['yeargen']), "r") From 5598f02c37a315a0a8b57dcc49fa25940bd5fe5a Mon Sep 17 00:00:00 2001 From: Harris Tzovanakis Date: Thu, 3 Dec 2015 17:28:43 +0100 Subject: [PATCH 50/59] elasticsearch: fallback file per hostname * Adds `CFG_ELASTICSEARCH_FALLBACK_DIRECTORY` configuration variable to specify where the fallback file to be stored. * Creates a fallback file with the hostname as file name. Signed-off-by: Harris Tzovanakis --- modules/miscutil/lib/elasticsearch_logging.py | 50 ++++++++++++++----- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/modules/miscutil/lib/elasticsearch_logging.py b/modules/miscutil/lib/elasticsearch_logging.py index be7ab306f9..d2308d6816 100644 --- a/modules/miscutil/lib/elasticsearch_logging.py +++ b/modules/miscutil/lib/elasticsearch_logging.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- ## ## This file is part of Invenio. -## Copyright (C) 2014 CERN. +## Copyright (C) 2014, 2015 CERN. ## ## Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as @@ -17,20 +17,21 @@ ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. -__revision__ = \ - "$Id$" - -from invenio.config import \ - CFG_ELASTICSEARCH_LOGGING, \ - CFG_ELASTICSEARCH_INDEX_PREFIX, \ - CFG_ELASTICSEARCH_HOSTS, \ - CFG_ELASTICSEARCH_SUFFIX_FORMAT, \ - CFG_ELASTICSEARCH_MAX_QUEUE_LENGTH, \ - CFG_ELASTICSEARCH_FLUSH_INTERVAL +from invenio.config import ( + CFG_ELASTICSEARCH_FALLBACK_DIRECTORY, + CFG_ELASTICSEARCH_FLUSH_INTERVAL, + CFG_ELASTICSEARCH_HOSTS, + CFG_ELASTICSEARCH_INDEX_PREFIX, + CFG_ELASTICSEARCH_LOGGING, + CFG_ELASTICSEARCH_MAX_QUEUE_LENGTH, + CFG_ELASTICSEARCH_SUFFIX_FORMAT, +) if CFG_ELASTICSEARCH_LOGGING: - import lumberjack import logging + import lumberjack + import os + import socket import sys def initialise_lumberjack(): @@ -49,6 +50,18 @@ def initialise_lumberjack(): else: config['interval'] = CFG_ELASTICSEARCH_FLUSH_INTERVAL + # Check if lumberjack directory exists + _lumberjack_exists(CFG_ELASTICSEARCH_FALLBACK_DIRECTORY) + + # Get hostname from socket (strip cern.ch) + fallback_file_name = socket.getfqdn().replace('.cern.ch', '') + + fallback_file_path = os.path.join( + CFG_ELASTICSEARCH_FALLBACK_DIRECTORY, + fallback_file_name + ) + config['fallback_log_file'] = fallback_file_path + lj = lumberjack.Lumberjack( hosts=CFG_ELASTICSEARCH_HOSTS, config=config) @@ -63,8 +76,21 @@ def initialise_lumberjack(): return lj + +def _lumberjack_exists(path): + """Check if lumberjack directory exists. + + :param str path: the path to lumberjack directory + """ + try: + os.makedirs(path) + except OSError: + if not os.path.isdir(path): + raise + LUMBERJACK = initialise_lumberjack() + def register_schema(*args, **kwargs): if not CFG_ELASTICSEARCH_LOGGING: return None From 0d10ee356d6e965f8aeee4f3c92d08355cd81c10 Mon Sep 17 00:00:00 2001 From: Harris Tzovanakis Date: Mon, 24 Aug 2015 15:19:18 +0200 Subject: [PATCH 51/59] webstat: downloads and pageviews as events * IMPROVEMENT Adds `downloads` and `pagesviews` as custom events. * Custom events will be registered only in elasticsearch if `CFG_ELASTICSEARCH_LOGGING` is enabled. Signed-off-by: Harris Tzovanakis --- modules/bibdocfile/lib/bibdocfile.py | 47 +++++----- .../lib/bibrank_downloads_similarity.py | 49 +++++------ .../invenio_2015_08_24_custom_events.py | 66 ++++++++++++++ modules/webstat/etc/webstat.cfg | 88 +++++++++++++------ modules/webstat/lib/webstat.py | 75 ++++++++-------- modules/webstat/lib/webstat_config.py | 69 +++++++-------- modules/webstat/lib/webstat_templates.py | 49 ++++++----- modules/webstat/lib/webstatadmin.py | 36 ++++---- 8 files changed, 287 insertions(+), 192 deletions(-) create mode 100644 modules/miscutil/lib/upgrades/invenio_2015_08_24_custom_events.py diff --git a/modules/bibdocfile/lib/bibdocfile.py b/modules/bibdocfile/lib/bibdocfile.py index 596b3b2526..49a6791927 100644 --- a/modules/bibdocfile/lib/bibdocfile.py +++ b/modules/bibdocfile/lib/bibdocfile.py @@ -1,5 +1,5 @@ # This file is part of Invenio. -# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 CERN. +# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 CERN. # # Invenio is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -95,6 +95,7 @@ from sets import Set as set # pylint: enable=W0622 +#from invenio.webstat import register_customevent from invenio.shellutils import escape_shell_arg, run_shell_command from invenio.dbquery import run_sql, DatabaseError from invenio.errorlib import register_exception @@ -123,7 +124,6 @@ CFG_BIBDOCFILE_ADDITIONAL_KNOWN_MIMETYPES, \ CFG_BIBDOCFILE_PREFERRED_MIMETYPES_MAPPING, \ CFG_BIBCATALOG_SYSTEM, \ - CFG_ELASTICSEARCH_LOGGING, \ CFG_ELASTICSEARCH_BOT_AGENT_STRINGS from invenio.bibcatalog import BIBCATALOG_SYSTEM from invenio.bibdocfile_config import CFG_BIBDOCFILE_ICON_SUBFORMAT_RE, \ @@ -132,11 +132,6 @@ import invenio.template -if CFG_ELASTICSEARCH_LOGGING: - import logging - - _DOWNLOAD_LOG = logging.getLogger('events.downloads') - def _plugin_bldr(dummy, plugin_code): """Preparing the plugin dictionary structure""" ret = {} @@ -2917,29 +2912,27 @@ def register_download(self, ip_address, version, docformat, user_agent, docformat = docformat.upper() if not version: version = self.get_latest_version() - if CFG_ELASTICSEARCH_LOGGING: - log_entry = { - 'id_bibrec': recid, - 'id_bibdoc': self.id, - 'file_version': version, - 'file_format': docformat, - 'id_user': userid, - 'client_host': ip_address, - 'user_agent': user_agent - } - if user_agent is not None: + + try: + from invenio.webstat import register_customevent + ## register event in webstat + download_register_event = [ + recid, self.id, version, docformat, userid, ip_address, + user_agent + ] + is_bot = False + if user_agent: for bot in CFG_ELASTICSEARCH_BOT_AGENT_STRINGS: if bot in user_agent: - log_entry['bot'] = True + is_bot = True break - _DOWNLOAD_LOG.info(log_entry) - else: - return run_sql("INSERT INTO rnkDOWNLOADS " - "(id_bibrec,id_bibdoc,file_version,file_format," - "id_user,client_host,download_time) VALUES " - "(%s,%s,%s,%s,%s,INET_ATON(%s),NOW())", - (recid, self.id, version, docformat, - userid, ip_address,)) + download_register_event.append(is_bot) + register_customevent("downloads", download_register_event) + except: + register_exception( + ("Do the webstat tables exists? Try with 'webstatadmin" + " --load-config'") + ) def get_incoming_relations(self, rel_type=None): """Return all relations in which this BibDoc appears on target position diff --git a/modules/bibrank/lib/bibrank_downloads_similarity.py b/modules/bibrank/lib/bibrank_downloads_similarity.py index 568cd0261b..5921cd013b 100644 --- a/modules/bibrank/lib/bibrank_downloads_similarity.py +++ b/modules/bibrank/lib/bibrank_downloads_similarity.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # This file is part of Invenio. -# Copyright (C) 2005, 2006, 2007, 2008, 2010, 2011, 2012 CERN. +# Copyright (C) 2005, 2006, 2007, 2008, 2010, 2011, 2012, 2015 CERN. # # Invenio is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -20,19 +20,14 @@ __revision__ = \ "$Id$" -from invenio.config import \ - CFG_ACCESS_CONTROL_LEVEL_SITE, \ - CFG_CERN_SITE, \ - CFG_ELASTICSEARCH_LOGGING, \ - CFG_ELASTICSEARCH_BOT_AGENT_STRINGS +from invenio.config import ( + CFG_ACCESS_CONTROL_LEVEL_SITE, CFG_CERN_SITE, + CFG_ELASTICSEARCH_BOT_AGENT_STRINGS +) from invenio.dbquery import run_sql from invenio.bibrank_downloads_indexer import database_tuples_to_single_list from invenio.search_engine_utils import get_fieldvalues - -if CFG_ELASTICSEARCH_LOGGING: - import logging - - _PAGEVIEW_LOG = logging.getLogger('events.pageviews') +from invenio.errorlib import register_exception def record_exists(recID): """Return 1 if record RECID exists. @@ -62,24 +57,26 @@ def register_page_view_event(recid, uid, client_ip_address, user_agent): # do not register access if we are in read-only access control # site mode: return [] - if CFG_ELASTICSEARCH_LOGGING: - log_event = { - 'id_bibrec': recid, - 'id_user': uid, - 'client_host': client_ip_address, - 'user_agent': user_agent - } - if user_agent is not None: + + # register event in webstat + try: + from invenio.webstat import register_customevent + pageviews_register_event = [ + recid, uid, client_ip_address, user_agent + ] + is_bot = False + if user_agent: for bot in CFG_ELASTICSEARCH_BOT_AGENT_STRINGS: if bot in user_agent: - log_event['bot'] = True + is_bot = True break - _PAGEVIEW_LOG.info(log_event) - else: - return run_sql("INSERT INTO rnkPAGEVIEWS " \ - " (id_bibrec,id_user,client_host,view_time) " \ - " VALUES (%s,%s,INET_ATON(%s),NOW())", \ - (recid, uid, client_ip_address)) + pageviews_register_event.append(is_bot) + register_customevent("pageviews", pageviews_register_event) + except: + register_exception( + ("Do the webstat tables exists? Try with 'webstatadmin" + " --load-config'") + ) def calculate_reading_similarity_list(recid, type="pageviews"): """Calculate reading similarity data to use in reading similarity diff --git a/modules/miscutil/lib/upgrades/invenio_2015_08_24_custom_events.py b/modules/miscutil/lib/upgrades/invenio_2015_08_24_custom_events.py new file mode 100644 index 0000000000..accc436cd1 --- /dev/null +++ b/modules/miscutil/lib/upgrades/invenio_2015_08_24_custom_events.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# +# This file is part of Invenio. +# Copyright (C) 2015 CERN. +# +# Invenio is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# Invenio is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Invenio; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +"""Add downloads and pageviews as custom events.""" + +from invenio.webstat import create_customevent + +depends_on = ['invenio_release_1_3_0'] + + +def info(): + """Return upgrade recipe information.""" + return "Adds downloads and pageviews as custom events." + + +def do_upgrade(): + """Carry out the upgrade.""" + # create the downloads + create_customevent( + event_id="downloads", + name="downloads", + cols=[ + "id_bibrec", "id_bibdoc", "file_version", "file_format", + "id_user", "client_host", "user_agent", "bot" + ] + ) + # create the pageviews + create_customevent( + event_id="pageviews", + name="pageviews", + cols=[ + "id_bibrec", "id_user", "client_host", "user_agent", "bot" + ] + ) + return 1 + + +def estimate(): + """Estimate running time of upgrade in seconds (optional).""" + return 1 + + +def pre_upgrade(): + """Pre-upgrade checks.""" + pass # because slashes would still work + + +def post_upgrade(): + """Post-upgrade checks.""" + pass diff --git a/modules/webstat/etc/webstat.cfg b/modules/webstat/etc/webstat.cfg index ed28f4bc9c..9e31615acd 100644 --- a/modules/webstat/etc/webstat.cfg +++ b/modules/webstat/etc/webstat.cfg @@ -4,14 +4,14 @@ [general] visitors_box = True search_box = True -record_box = True -bibsched_box = True +record_box = False +bibsched_box = False basket_box = True alert_box = True -loan_box = True apache_box = True uptime_box = True -waiting_box = True +loan_box = True +waiting_box = False max_ingestion_health = 25 [webstat_custom_event_1] @@ -27,43 +27,79 @@ param2 = alert param3 = user [webstat_custom_event_3] -name = journals -param1 = action -param2 = journal_name -param3 = issue_number -param4 = category -param5 = language -param6 = articleid +name = media_download +param1 = file +param2 = type +param3 = format +param4 = ip [webstat_custom_event_4] +name = ejournal +param1 = publication +param2 = volume +param3 = year +param4 = page +param5 = source +param6 = ip + +[webstat_custom_event_5] +name = media_view +param1 = file +param2 = type +param3 = format +param4 = ip +param5 = external +param6 = recid + +[webstat_custom_event_6] name = websubmissions param1 = doctype -[webstat_custom_event_5] +[webstat_custom_event_7] name = loanrequest param1 = request_id param2 = loan_id -[webstat_custom_event_6] -name = login -param1 = IP -param2 = UID -param3 = email +[webstat_custom_event_8] +name = journals +param1 = action +param2 = journal_name +param3 = issue_number +param4 = category +param5 = language +param6 = articleid -[webstat_custom_event_7] -name = apikeyusage +[webstat_custom_event_9] +name = ebidding_opening param1 = user_id -param2 = key_id -param3 = path -param4 = query +param2 = etendering_ref +param3 = action + +[webstat_custom_event_10] +name = downloads +param1 = id_bibrec +param2 = id_bibdoc +param3 = file_version +param4 = file_format +param5 = id_user +param6 = client_host +param7 = user_agent +param7 = bot + +[webstat_custom_event_11] +name = pageviews +param1 = id_bibrec +param2 = id_user +param3 = client_host +param4 = user_agent +param5 = bot [apache_log_analyzer] profile = nil nb-histogram-items-to-print = 20 -exclude-ip-list = ("137.138.249.162" "137.138.246.86") -home-collection = "Atlantis Institute of Fictive Science" -search-interface-url = "/collection/" -search-interface-url-old-style = "/?" +exclude-ip-list = '("137.138.198.205") +home-collection = "CERN Document Server" +search-interface-url = "/?" detailed-record-url = "/record/" search-engine-url = "/search?" search-engine-url-old-style = "/search.py?" diff --git a/modules/webstat/lib/webstat.py b/modules/webstat/lib/webstat.py index de7b771346..9f0316da4b 100644 --- a/modules/webstat/lib/webstat.py +++ b/modules/webstat/lib/webstat.py @@ -1,19 +1,19 @@ -# This file is part of Invenio. -# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2013 CERN. -# -# Invenio is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of the -# License, or (at your option) any later version. -# -# Invenio is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Invenio; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. +## This file is part of Invenio. +## Copyright (C) 2007, 2008, 2009, 2010, 2011, 2013 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. __revision__ = "$Id$" __lastupdated__ = "$Date$" @@ -796,31 +796,14 @@ def register_customevent(event_id, *arguments): # Make sql query if len(arguments[0]) != 0: - sql_param = [] - sql_query = ["INSERT INTO %s (" % wash_table_column_name(tbl_name)] - for title in col_titles: - sql_query.append("`%s`" % title) - sql_query.append(",") - # del the last ',' - sql_query.pop() - sql_query.append(") VALUES (") - for argument in arguments[0]: - sql_query.append("%s") - sql_query.append(",") - sql_param.append(argument) - # del the last ',' - sql_query.pop() - sql_query.append(")") - sql_str = ''.join(sql_query) - run_sql(sql_str, tuple(sql_param)) - + # Register the event on elastic search ONLY! + elastic_search_event_id = "events.{0}".format(event_id) # Register the event on elastic search - if CFG_ELASTICSEARCH_LOGGING and event_id in \ + if CFG_ELASTICSEARCH_LOGGING and elastic_search_event_id in \ CFG_ELASTICSEARCH_EVENTS_MAP.keys(): # Initialize elastic search handler elastic_search_parameters = zip(col_titles, arguments[0]) - event_logger_name = "events.{0}".format(event_id) - logger = getLogger(event_logger_name) + logger = getLogger(elastic_search_event_id) log_event = {} for key, value in elastic_search_parameters: log_event[key] = value @@ -829,6 +812,24 @@ def register_customevent(event_id, *arguments): else: postprocessors = [] logger.info(log_event, {'postprocessors': postprocessors}) + else: + sql_param = [] + sql_query = ["INSERT INTO %s (" % wash_table_column_name(tbl_name)] + for title in col_titles: + sql_query.append("`%s`" % title) + sql_query.append(",") + # del the last ',' + sql_query.pop() + sql_query.append(") VALUES (") + for argument in arguments[0]: + sql_query.append("%s") + sql_query.append(",") + sql_param.append(argument) + # del the last ',' + sql_query.pop() + sql_query.append(")") + sql_str = ''.join(sql_query) + run_sql(sql_str, tuple(sql_param)) else: # kwalitee: disable=sql run_sql( diff --git a/modules/webstat/lib/webstat_config.py b/modules/webstat/lib/webstat_config.py index da93dc732d..363d084278 100644 --- a/modules/webstat/lib/webstat_config.py +++ b/modules/webstat/lib/webstat_config.py @@ -1,50 +1,47 @@ # -*- coding: utf-8 -*- -# -# This file is part of Invenio. -# Copyright (C) 2008, 2010, 2011 CERN. -# -# Invenio is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of the -# License, or (at your option) any later version. -# -# Invenio is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Invenio; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. +## +## This file is part of Invenio. +## Copyright (C) 2008, 2010, 2011, 2015 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Invenio WebStat Configuration.""" __revision__ = "$Id$" -from invenio.config import CFG_ETCDIR, CFG_ELASTICSEARCH_LOGGING, CFG_CERN_SITE +from invenio.config import ( + CFG_ETCDIR, + CFG_ELASTICSEARCH_LOGGING, + CFG_CERN_SITE +) CFG_WEBSTAT_CONFIG_PATH = CFG_ETCDIR + "/webstat/webstat.cfg" if CFG_CERN_SITE: CFG_ELASTICSEARCH_EVENTS_MAP = { - "events.loanrequest": { - "_source": { - "enabled": True - }, - "properties": { - "request_id": { - "type": "integer" - }, - "load_id": { - "type": "integer" - } - } - } + "events.alerts": True, + "events.baskets": True, + "events.downloads": True, + "events.ebidding_opening": True, + "events.ejournal": True, + "events.journals": True, + "events.loanrequest": True, + "events.media_download": True, + "events.media_view": True, + "events.pageviews": True, + "events.websubmissions": True, } else: CFG_ELASTICSEARCH_EVENTS_MAP = {} - -if CFG_ELASTICSEARCH_LOGGING: - from invenio.elasticsearch_logging import register_schema - for name, arguments in CFG_ELASTICSEARCH_EVENTS_MAP.items(): - register_schema(name, arguments) diff --git a/modules/webstat/lib/webstat_templates.py b/modules/webstat/lib/webstat_templates.py index 83250384e9..f64c730276 100644 --- a/modules/webstat/lib/webstat_templates.py +++ b/modules/webstat/lib/webstat_templates.py @@ -1,30 +1,29 @@ -# This file is part of Invenio. -# Copyright (C) 2007, 2008, 2010, 2011, 2013 CERN. -# -# Invenio is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of the -# License, or (at your option) any later version. -# -# Invenio is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Invenio; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. +## This file is part of Invenio. +## Copyright (C) 2007, 2008, 2010, 2011, 2013, 2015 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. __revision__ = "$Id$" __lastupdated__ = "$Date$" import datetime, cgi, urllib, os import re -from invenio.config import \ - CFG_WEBDIR, \ - CFG_SITE_URL, \ - CFG_SITE_LANG, \ - CFG_SITE_NAME +from invenio.config import ( + CFG_WEBDIR, CFG_SITE_URL, CFG_SITE_LANG, CFG_SITE_NAME, + CFG_ELASTICSEARCH_LOGGING +) from invenio.search_engine import get_coll_sons from invenio.webstat_engine import get_invenio_error_details @@ -148,7 +147,13 @@ def tmpl_customevent_list(self, customevents, ln=CFG_SITE_LANG): When a custom event has been made available, it is displayed below.

    """ % CFG_SITE_URL - + # If elasticsearch is enabled display the message + if CFG_ELASTICSEARCH_LOGGING: + message = ( + "Elastic search logging for custom events has been enabled " + "the custom events will no longer been displayed here." + ) + return out + message temp_out = "" for event in customevents: temp_out += """
  • %s
    • """ \ diff --git a/modules/webstat/lib/webstatadmin.py b/modules/webstat/lib/webstatadmin.py index 362a36de8e..534092b73f 100644 --- a/modules/webstat/lib/webstatadmin.py +++ b/modules/webstat/lib/webstatadmin.py @@ -1,21 +1,21 @@ -# $id: webstatadmin.py,v 1.28 2007/04/01 23:46:46 tibor exp $ -# -# This file is part of Invenio. -# Copyright (C) 2007, 2008, 2010, 2011 CERN. -# -# Invenio is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of the -# License, or (at your option) any later version. -# -# Invenio is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Invenio; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. +## $id: webstatadmin.py,v 1.28 2007/04/01 23:46:46 tibor exp $ +## +## This file is part of Invenio. +## Copyright (C) 2007, 2008, 2010, 2011 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. __revision__ = "$Id$" __lastupdated__ = "$Date$" From bd7c03a8c3254036ae70725dc42202e80d98351a Mon Sep 17 00:00:00 2001 From: Martin Vesper Date: Wed, 18 Nov 2015 14:32:19 +0100 Subject: [PATCH 52/59] BibCirculation: reset ILL overdue_letter_number * A ILL is currently extended by manually updating the *due_date* of that ILL, this leads to the following issue: A user receives an ILL recall letter, therefore gets the ILL extended at the library. The next ILL recall letter will be of the second category, since the *overdue_letter_number* never gets reseted. * Introduces the new function *update_ill_request_letter_number* * Changes the *overdue_letter_number* in case the updated due_date is a later date than the current due_date in *ill_request_details_step2*. Signed-off-by: Martin Vesper --- .../lib/bibcirculation_dblayer.py | 5 +++++ .../lib/bibcirculationadminlib.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/modules/bibcirculation/lib/bibcirculation_dblayer.py b/modules/bibcirculation/lib/bibcirculation_dblayer.py index 4ddf5a97ce..3141505563 100644 --- a/modules/bibcirculation/lib/bibcirculation_dblayer.py +++ b/modules/bibcirculation/lib/bibcirculation_dblayer.py @@ -2721,6 +2721,11 @@ def get_purchase_request_borrower_details(ill_request_id): else: return None +def update_ill_request_letter_number(ill_request_id, overdue_letter_number): + query = ('UPDATE crcILLREQUEST set overdue_letter_number=%s ' + 'where id=%s') + run_sql(query, (overdue_letter_number, ill_request_id)) + def update_ill_request(ill_request_id, library_id, request_date, expected_date, arrival_date, due_date, return_date, status, cost, barcode, library_notes): diff --git a/modules/bibcirculation/lib/bibcirculationadminlib.py b/modules/bibcirculation/lib/bibcirculationadminlib.py index 0d872c079e..11de44bc5f 100644 --- a/modules/bibcirculation/lib/bibcirculationadminlib.py +++ b/modules/bibcirculation/lib/bibcirculationadminlib.py @@ -4969,6 +4969,25 @@ def ill_request_details_step2(req, delete_key, ill_request_id, new_status, barcode = db.get_ill_barcode(ill_request_id) db.update_ill_loan_status(borrower_id, barcode, return_date, 'ill') + # ill recall letter issue + try: + from invenio.dbquery import run_sql + _query = ('SELECT due_date from crcILLREQUEST where id = %s') + _due = run_sql(_query, (ill_request_id))[0][0] + + # Since we don't know if the due_date is a string or datetime + try: + _due_date = datetime.datetime.strptime(due_date, '%Y-%m-%d') + except TypeError: + _due_date = due_date + + # This means that the ILL got extended, we therefore reset the + # overdue_letter_numer + if _due < _due_date: + db.update_ill_request_letter_number(ill_request_id, 0) + except Exception: + pass + db.update_ill_request(ill_request_id, library_id, request_date, expected_date, arrival_date, due_date, return_date, new_status, cost, barcode, From a32bca836efe24e7e137c065b828958753a6928e Mon Sep 17 00:00:00 2001 From: David Zerulla Date: Wed, 2 Sep 2015 13:46:24 +0200 Subject: [PATCH 53/59] elasticsearch: Fix invenio.config import * Fixes creation of the Invenio config with the tool inveniocfg. The `elasticsearch_logging` module tries to use the invenio.config before the config is created. To reproduce remove the invenio.config and run `inveniocfg --update-all`. Signed-off-by: David Zerulla --- modules/miscutil/lib/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/miscutil/lib/__init__.py b/modules/miscutil/lib/__init__.py index baa980c8e8..4862269232 100644 --- a/modules/miscutil/lib/__init__.py +++ b/modules/miscutil/lib/__init__.py @@ -27,4 +27,9 @@ ## Because we use getLogger calls to do logging, handlers aren't initialised ## unless we explicitly call/import this code somewhere. -import elasticsearch_logging +# TODO: This should be done in an other way! +# The import breaks if inveniocfg creates the invenio config. +try: + import elasticsearch_logging +except ImportError as e: + pass From eae3331e4a122d71368c6c23cd1d321fcf6b0e0a Mon Sep 17 00:00:00 2001 From: Sebastian Witowski Date: Tue, 17 May 2016 10:24:49 +0200 Subject: [PATCH 54/59] Bibfield: Add y subfield to FFT * FFT__y subfield contains language for files uploaded to the committee documents collections. Signed-off-by: Sebastian Witowski --- modules/bibfield/etc/atlantis.cfg | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/bibfield/etc/atlantis.cfg b/modules/bibfield/etc/atlantis.cfg index 217a13e17a..d2fa3a3d79 100644 --- a/modules/bibfield/etc/atlantis.cfg +++ b/modules/bibfield/etc/atlantis.cfg @@ -400,6 +400,7 @@ fft: ("FFT__t", "docfile_type"), ("FFT__v", "version"), ("FFT__x", "icon_path"), + ("FFT__y", "link_text"), ("FFT__z", "comment"), ("FFT__w", "document_moreinfo"), ("FFT__p", "version_moreinfo"), @@ -416,6 +417,7 @@ fft: 'docfile_type': value['t'], 'version': value['v'], 'icon_path': value['x'], + 'link_text': value['y'], 'comment': value['z'], 'document_moreinfo': value['w'], 'version_moreinfo': value['p'], @@ -438,7 +440,7 @@ fft: 'description':value['y'], 'comment':value['z']} producer: - json_for_marc(), {"FFT__a": "path", "FFT__d": "description", "FFT__f": "eformat", "FFT__i": "temporary_id", "FFT__m": "new_name", "FFT__o": "flag", "FFT__r": "restriction", "FFT__s": "timestamp", "FFT__t": "docfile_type", "FFT__v": "version", "FFT__x": "icon_path", "FFT__z": "comment", "FFT__w": "document_moreinfo", "FFT__p": "version_moreinfo", "FFT__b": "version_format_moreinfo", "FFT__f": "format_moreinfo"} + json_for_marc(), {"FFT__a": "path", "FFT__d": "description", "FFT__f": "eformat", "FFT__i": "temporary_id", "FFT__m": "new_name", "FFT__o": "flag", "FFT__r": "restriction", "FFT__s": "timestamp", "FFT__t": "docfile_type", "FFT__v": "version", "FFT__x": "icon_path", "FFT__y": "link_text", "FFT__z": "comment", "FFT__w": "document_moreinfo", "FFT__p": "version_moreinfo", "FFT__b": "version_format_moreinfo", "FFT__f": "format_moreinfo"} funding_info: creator: From 6c03a66b28b30cf4bd49b3b4fad99a49fb0a10ff Mon Sep 17 00:00:00 2001 From: Ludmila Marian Date: Tue, 31 May 2016 10:05:56 +0200 Subject: [PATCH 55/59] WebSearch: improve detection of record viewers * If CFG_CERN_SITE, consider the possibility of the viewer of a record being an egroup, defined as 'foo@cern.ch' insted of 'foo [CERN]'. Signed-off-by: Ludmila Marian --- modules/websearch/lib/search_engine.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/websearch/lib/search_engine.py b/modules/websearch/lib/search_engine.py index 73a77522a7..9e79bcb741 100644 --- a/modules/websearch/lib/search_engine.py +++ b/modules/websearch/lib/search_engine.py @@ -358,6 +358,10 @@ def is_user_viewer_of_record(user_info, recid): email = email_or_group.strip().lower() if user_info['email'].strip().lower() == email: return True + if CFG_CERN_SITE: + #the egroup might be in the form egroup@cern.ch + if email_or_group.replace('@cern.ch', ' [CERN]') in user_info['group']: + return True return False def check_user_can_view_record(user_info, recid): From ae6976481bdad4bf5e6786d0806e6dfdc8fbe4ec Mon Sep 17 00:00:00 2001 From: Nikolaos Kasioumis Date: Fri, 23 Nov 2012 16:38:10 +0100 Subject: [PATCH 56/59] BibSword: Major rewrite of the module --- configure.ac | 1 + modules/bibsword/lib/Makefile.am | 10 +- modules/bibsword/lib/bibsword_client.py | 3281 +++++++++-------- .../bibsword/lib/bibsword_client_dblayer.py | 701 ++-- .../bibsword/lib/bibsword_client_formatter.py | 1196 ------ modules/bibsword/lib/bibsword_client_http.py | 188 - .../bibsword/lib/bibsword_client_server.py | 521 +++ .../bibsword/lib/bibsword_client_templates.py | 3057 +++++++++------ .../bibsword/lib/bibsword_client_tester.py | 668 ---- modules/bibsword/lib/bibsword_config.py | 196 +- modules/bibsword/lib/bibsword_webinterface.py | 935 +++-- .../bibsword/lib/client_servers/Makefile.am | 24 + .../bibsword/lib/client_servers/__init__.py | 0 .../bibsword/lib/client_servers/arxiv_org.py | 846 +++++ modules/webstyle/css/invenio.css | 257 ++ modules/webstyle/lib/webinterface_layout.py | 11 +- 16 files changed, 6232 insertions(+), 5660 deletions(-) delete mode 100644 modules/bibsword/lib/bibsword_client_formatter.py delete mode 100644 modules/bibsword/lib/bibsword_client_http.py create mode 100644 modules/bibsword/lib/bibsword_client_server.py delete mode 100644 modules/bibsword/lib/bibsword_client_tester.py create mode 100644 modules/bibsword/lib/client_servers/Makefile.am create mode 100644 modules/bibsword/lib/client_servers/__init__.py create mode 100644 modules/bibsword/lib/client_servers/arxiv_org.py diff --git a/configure.ac b/configure.ac index 5a17e890bf..ad282c49f7 100644 --- a/configure.ac +++ b/configure.ac @@ -775,6 +775,7 @@ AC_CONFIG_FILES([config.nice \ modules/bibsword/doc/admin/Makefile \ modules/bibsword/doc/hacking/Makefile \ modules/bibsword/lib/Makefile \ + modules/bibsword/lib/client_servers/Makefile \ modules/bibsword/etc/Makefile \ modules/bibupload/bin/Makefile \ modules/bibupload/bin/bibupload \ diff --git a/modules/bibsword/lib/Makefile.am b/modules/bibsword/lib/Makefile.am index 359b78c2cc..07ad574a0d 100644 --- a/modules/bibsword/lib/Makefile.am +++ b/modules/bibsword/lib/Makefile.am @@ -15,16 +15,16 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. +SUBDIRS = client_servers + pylibdir = $(libdir)/python/invenio pylib_DATA = bibsword_config.py \ - bibsword_client_dblayer.py \ - bibsword_client_formatter.py \ - bibsword_client_http.py \ bibsword_client.py \ + bibsword_client_dblayer.py \ + bibsword_client_server.py \ bibsword_client_templates.py \ - bibsword_webinterface.py \ - bibsword_client_tester.py + bibsword_webinterface.py EXTRA_DIST = $(pylib_DATA) diff --git a/modules/bibsword/lib/bibsword_client.py b/modules/bibsword/lib/bibsword_client.py index d1412f3ff2..7de1b5e8d8 100644 --- a/modules/bibsword/lib/bibsword_client.py +++ b/modules/bibsword/lib/bibsword_client.py @@ -1,5 +1,7 @@ +# -*- coding: utf-8 -*- +# # This file is part of Invenio. -# Copyright (C) 2010, 2011 CERN. +# Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016 CERN. # # Invenio is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -15,1697 +17,1740 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. -''' -BibSWORD Client Engine -''' -import getopt -import sys -import datetime -import time -import StringIO -from tempfile import NamedTemporaryFile -from invenio.bibsword_config import CFG_SUBMISSION_STATUS_SUBMITTED, \ - CFG_SUBMISSION_STATUS_REMOVED, \ - CFG_SUBMISSION_STATUS_PUBLISHED, \ - CFG_BIBSWORD_SERVICEDOCUMENT_UPDATE_TIME -from invenio.bibsword_client_http import RemoteSwordServer -from invenio.bibsword_client_formatter import format_remote_server_infos, \ - format_remote_collection, \ - format_collection_informations, \ - format_primary_categories, \ - format_secondary_categories, \ - get_media_from_recid, \ - get_medias_to_submit, \ - format_file_to_zip_archiv, \ - format_marcxml_file, \ - format_link_from_result, \ - get_report_number_from_macrxml, \ - format_links_from_submission, \ - format_id_from_submission, \ - update_marcxml_with_remote_id, \ - ArXivFormat, \ - format_submission_status, \ - format_author_from_marcxml, \ - upload_fulltext, \ - update_marcxml_with_info -from invenio.bibsword_client_dblayer import get_all_remote_server, \ - get_last_update, \ - update_servicedocument, \ - select_servicedocument, \ - get_remote_server_auth, \ - is_record_sent_to_server, \ - select_submitted_record_infos, \ - update_submission_status, \ - insert_into_swr_clientdata, \ - count_nb_submitted_record, \ - select_remote_server_infos -from invenio.bibformat import record_get_xml -from invenio.bibsword_client_templates import BibSwordTemplate -from invenio.config import CFG_TMPDIR, CFG_SITE_ADMIN_EMAIL - -#------------------------------------------------------------------------------- -# Implementation of the BibSword API -#------------------------------------------------------------------------------- - -def list_remote_servers(id_server=''): - ''' - Get the list of remote servers implemented by the Invenio SWORD API. - @return: list of tuples [ { 'id', 'name' } ] - ''' - - return get_all_remote_server(id_server) - - -def list_server_info(id_server): - ''' - Get all informations about the server's options such as SWORD version, - maxUploadSize, ... These informations are found in the servicedocument of - the given server. - @param id_server: #id of the server in the table swrREMOTESERVER - @return: tuple { 'version', 'maxUploadSize', 'verbose', 'noOp' } - ''' - - service = get_servicedocument(id_server) - return format_remote_server_infos(service) - - -def list_collections_from_server(id_server): - ''' - List all the collections found in the servicedocument of the given server. - @param id_server: #id of the server in the table swrRMOTESERVER - @return: list of information's tuples [ { 'id', 'label', 'url' } ] - ''' - - service = get_servicedocument(id_server) - if service == '': - return '' - return format_remote_collection(service) - - -def list_collection_informations(id_server, id_collection): - ''' - List all information concerning the collection such as the list of - accepted type of media, if the collection allows mediation, if the - collection accept packaging, ... - @param id_server: #id of the server in the table swrRMOTESERVER - @param id_collection: id of the collection found in collection listing - @return: information's tuple: {[accept], 'collectionPolicy', 'mediation', - 'treatment', 'acceptPackaging'} - ''' - - service = get_servicedocument(id_server) - if service == '': - return '' - return format_collection_informations(service, id_collection) - - -def list_mandated_categories(id_server, id_collection): - ''' - The mandated categories are the categories that must be specified to the - remote server's collection. - In some SWORD implementation, they are not used but in some they do. - @param id_server: #id of the server in the table swrRMOTESERVER - @param id_collection: id of the collection found by listing them - @return: list of category's tuples [ { 'id', 'label', 'url' } ] - ''' - - service = get_servicedocument(id_server) - if service == '': - return '' - return format_primary_categories(service, id_collection) - - -def list_optional_categories(id_server, id_collection): - ''' - The optional categories are only used as search option to retrieve the - resource. - @param id_server: #id of the server in the table swrRMOTESERVER - @param id_collection: id of the collection found by listing them - @return: list of category's tuples [ { 'id', 'label', 'url' } ] - ''' - - service = get_servicedocument(id_server) - if service == '': - return '' - return format_secondary_categories(service, id_collection) - - -def list_submitted_resources(first_row, offset, action="submitted"): - ''' - List the swrCLIENTDATA table informations such as submitter, date of submission, - link to the resource and status of the submission. - It is possible to limit the amount of result by specifing a remote server, - the id of the bibRecord or both - @return: list of submission's tuple [ { 'id', 'links', 'type, 'submiter', - 'date', 'status'} ] - ''' - - #get all submission from the database - if action == 'submitted': - submissions = select_submitted_record_infos(first_row, offset) - else: - nb_submission = count_nb_submitted_record() - submissions = select_submitted_record_infos(0, nb_submission) - - authentication_info = get_remote_server_auth(1) - connection = RemoteSwordServer(authentication_info) - - #retrieve the status of all submission and update it if necessary - for submission in submissions: - if action == 'submitted' and submission['status'] != \ - CFG_SUBMISSION_STATUS_SUBMITTED: - continue - status_xml = connection.get_submission_status(submission['link_status']) - if status_xml != '': - status = format_submission_status(status_xml) - if status['status'] != submission['status']: - update_submission_status(submission['id'], - status['status'], - status['id_submission']) - - if status['status'] == CFG_SUBMISSION_STATUS_PUBLISHED: - update_marcxml_with_remote_id(submission['id_record'], - submission['id_remote']) - if status['status'] == CFG_SUBMISSION_STATUS_REMOVED: - update_marcxml_with_remote_id(submission['id_record'], - submission['id_remote'], - "delete") - update_marcxml_with_info(submission['id_record'], - submission['user_name'], - submission['submission_date'], - submission['id_remote'], "delete") - - return select_submitted_record_infos(first_row, offset) - - -def get_marcxml_from_record(recid): - ''' - Return a string containing the metadata in the format of a marcxml file. - The marcxml is retreived by using the given record id. - @param recid: id of the record to be retreive on the database - @return: string containing the marcxml file of the record - ''' - - return record_get_xml(recid) - - -def get_media_list(recid, selected_medias=None): - ''' - Parse the marcxml file to retrieve the link toward the media. Get every - media through its URL and set each of them and their type in a list of - tuple. - @param recid: recid of the record to consider - @return: list of tuples: [ { 'link', 'type', 'file' } ] - ''' - - if selected_medias == None: - selected_medias = [] - - medias = get_media_from_recid(recid) - - for media in medias: - for selected_media in selected_medias: - if selected_media == media['path']: - media['selected'] = 'checked="yes"' - selected_medias.remove(selected_media) - break - - - for selected_media in selected_medias: - - media = {} - media['path'] = selected_media - media['file'] = open(selected_media, 'r').read() - media['size'] = str(len(media['file'])) - - if selected_media.endswith('pdf'): - media['type'] = 'application/pdf' - elif selected_media.endswith('zip'): - media['type'] = 'application/zip' - elif selected_media.endswith('tar'): - media['type'] = 'application/tar' - elif selected_media.endswith('docx'): - media['type'] = 'application/docx' - elif selected_media.endswith('pdf'): - media['type'] = 'application/pdf' - else: - media['type'] = '' - - media['loaded'] = True - media['selected'] = 'checked="yes"' - medias.append(media) - - return medias - - -def compress_media_file(media_file_list): - ''' - Compress each file of the given list in a single zipped archive and return - this archive in a new media file list containing only one tuple. - @param media_file_list: list of tuple [ { 'type', 'file' } ] - @return: list containing only one tuple { 'type=zip', 'file=archive' } - ''' - - filelist = [] - - - - return format_file_to_zip_archiv(filelist) - - -def deposit_media(server_id, media, deposit_url, username='', - email=''): - ''' - Deposit all media containing in the given list in the deposit_url (usually - the url of the selected collection. A user name and password could be - selected if the submission is made 'on behalf of' an author - @param server_id: id of the remote server to deposit media - @param media: list of tuple [ { 'type', 'file' } ] - @param deposit_url: url of the deposition on the internet - @param username: name of the user depositing 'on behalf of' an author - @param email: allow user to get an acknowledgement of the deposit - @return: list of xml result file (could'd be sword error xml file) - ''' - - response = {'result': [], 'error': ''} - - authentication_info = get_remote_server_auth(server_id) - - if authentication_info['error'] != '': - return authentication_info['error'] - - connection = RemoteSwordServer(authentication_info) - if username != '' and email != '': - onbehalf = '''"%s" <%s>''' % (username, email) - else: - onbehalf = '' - - result = connection.deposit_media(media, deposit_url, onbehalf) - - return result - - -def format_metadata(marcxml, deposit_result, user_info, metadata=None): - ''' - Format an xml atom entry containing the metadata for the submission and - the list of url where the media have been deposited. - @param deposit_result: list of obtained response during deposition - @param marcxml: marc file where to find metadata - @param metadata: optionaly give other metadata that those from marcxml - @return: xml atom entry containing foramtted metadata and links - ''' - - if metadata == None: - metadata = {} - - # retrive all metadata from marcxml file - metadata_from_marcxml = format_marcxml_file(marcxml) - metadata['error'] = [] - - - #--------------------------------------------------------------------------- - # get the author name and email of the document (mandatory) - #--------------------------------------------------------------------------- - - if 'author_name' not in metadata: - if 'nickname' not in user_info: - metadata['error'].append("No submitter name given !") - metadata['author_name'] = '' - elif user_info['nickname'] == '': - metadata['error'].append("No submitter name given !") - else: - metadata['author_name'] = user_info['nickname'] - - if 'author_email' not in metadata: - if 'email' not in user_info: - metadata['error'].append("No submitter email given !") - metadata['author_email'] = '' - elif user_info['email'] == '': - metadata['error'].append("No submitter email given !") +""" +BibSWORD Client Library. +""" + +import cPickle +import json +import os +import random +import re + +from invenio.bibdocfile import BibRecDocs +from invenio.bibsword_config import( + CFG_BIBSWORD_CLIENT_SERVERS_PATH, + CFG_BIBSWORD_LOCAL_TIMEZONE, + CFG_BIBSWORD_MARC_FIELDS, + CFG_BIBSWORD_MAXIMUM_NUMBER_OF_CONTRIBUTORS, + CFG_BIBSWORD_UPDATED_DATE_MYSQL_FORMAT +) +import invenio.bibsword_client_dblayer as sword_client_db +from invenio.bibsword_client_templates import TemplateSwordClient +from invenio.config import CFG_PYLIBDIR +from invenio.errorlib import raise_exception +from invenio.messages import gettext_set_language +from invenio.pluginutils import PluginContainer +from invenio.search_engine import( + get_modification_date, + get_record, + record_exists +) + +sword_client_template = TemplateSwordClient() + +RANDOM_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +RANDOM_RANGE = range(32) + +_CLIENT_SERVERS = PluginContainer( + os.path.join( + CFG_PYLIBDIR, + 'invenio', + CFG_BIBSWORD_CLIENT_SERVERS_PATH, + '*.py' + ) +) + + +class BibSwordSubmission(object): + """The BibSwordSubmission class.""" + + def __init__( + self, + recid, + uid, + sid=None + ): + """ + The constructor. Assign the given + or a unique temporary submission id. + """ + + self._set_recid_and_record(recid) + self._set_sid(sid) + self._set_uid(uid) + + def _set_uid(self, uid): + self._uid = uid + + def get_uid(self): + return self._uid + + def get_sid(self): + return self._sid + + def _set_sid(self, sid=None): + if sid is None: + self._sid = self._get_random_id() else: - metadata['author_email'] = user_info['email'] - - - #--------------------------------------------------------------------------- - # get url and label of the primary category of the document (mandatory) - #--------------------------------------------------------------------------- + self._sid = sid + return self._sid - if 'primary_label' not in metadata: - metadata['error'].append('No primary category label given !') - metadata['primary_label'] = '' - elif metadata['primary_label'] == '': - metadata['error'].append('No primary category label given !') + def _get_random_id(self, size_range=RANDOM_RANGE, chars=RANDOM_CHARS): + return ''.join(random.choice(chars) for i in size_range) - if 'primary_url' not in metadata: - metadata['error'].append('No primary category url given !') - metadata['primary_url'] = '' - elif metadata['primary_url'] == '': - metadata['error'].append('No primary category url given !') - - - #--------------------------------------------------------------------------- - # get the link to the deposited fulltext of the document (mandatory) - #--------------------------------------------------------------------------- - - if deposit_result in ([], ''): - metadata['error'].append('No links to the media deposit found !') - metadata['links'] = [] - else: - metadata['links'] = format_link_from_result(deposit_result) - - - #--------------------------------------------------------------------------- - # get the id of the document (mandatory) - #--------------------------------------------------------------------------- - - if 'id' not in metadata: - if 'id' not in metadata_from_marcxml: - metadata['error'].append("No document id given !") - metadata['id'] = '' - elif metadata_from_marcxml['id'] == '': - metadata['error'].append("No document id given !") - metadata['id'] = '' - else: - metadata['id'] = metadata_from_marcxml['id'] - elif metadata['id'] == '': - metadata['error'].append("No document id given !") - - - #--------------------------------------------------------------------------- - # get the title of the document (mandatory) - #--------------------------------------------------------------------------- - - if 'title' not in metadata: - if 'title' not in metadata_from_marcxml or \ - not metadata_from_marcxml['title']: - metadata['error'].append("No title given !") - metadata['title'] = '' - else: - metadata['title'] = metadata_from_marcxml['title'] - elif metadata['title'] == '': - metadata['error'].append("No title given !") - - - #--------------------------------------------------------------------------- - # get the contributors of the document (mandatory) - #--------------------------------------------------------------------------- - - contributors = [] - if 'contributors' not in metadata: - if 'contributors' not in metadata_from_marcxml: - metadata['error'].append('No author given !') - elif metadata_from_marcxml['contributors'] == '': - metadata['error'].append('No author given !') - elif len(metadata_from_marcxml['contributors']) == 0: - metadata['error'].append('No author given !') - - else: - for contributor in metadata_from_marcxml['contributors']: - if contributor != '': - contributors.append(contributor) - if len(contributors) == 0: - metadata['error'].append('No author given !') + def _set_recid_and_record(self, recid): + self._recid = recid + self._record = get_record(recid) + + def get_recid(self): + return self._recid + + def _get_marc_field_parts(self, field): + tag = field[0:3].lower() + ind1 = field[3].lower().replace('_', ' ') + ind2 = field[4].lower().replace('_', ' ') + subfield = field[5].lower() + + return (tag, ind1, ind2, subfield) + + def get_record_title(self, force=False): + + if '_title' not in self.__dict__.keys() or force: + + self._title = "" + + (tag, ind1, ind2, subfield) = self._get_marc_field_parts( + CFG_BIBSWORD_MARC_FIELDS['title'] + ) + + for record_title in self._record.get(tag, []): + if _decapitalize(record_title[1:3]) == (ind1, ind2): + for (code, value) in record_title[0]: + if code.lower() == subfield: + self._title = value + break + # Only get the first occurrence + break + + return self._title + + def set_record_title(self, title): + # TODO: Should we unescape here? + self._title = title + + def get_record_abstract(self, force=False): + + if '_abstract' not in self.__dict__.keys() or force: + + self._abstract = "" + + (tag, ind1, ind2, subfield) = self._get_marc_field_parts( + CFG_BIBSWORD_MARC_FIELDS['abstract'] + ) + + for record_abstract in self._record.get(tag, []): + if _decapitalize(record_abstract[1:3]) == (ind1, ind2): + for (code, value) in record_abstract[0]: + if code.lower() == subfield: + self._abstract = value + break + # Only get the first occurrence + break + + return self._abstract + + def set_record_abstract(self, abstract): + self._abstract = abstract + + def get_record_author(self, force=False): + + if '_author' not in self.__dict__.keys() or force: + + author_name = "" + author_email = "" + author_affiliation = "" + + ( + tag_name, + ind1_name, + ind2_name, + subfield_name + ) = self._get_marc_field_parts( + CFG_BIBSWORD_MARC_FIELDS['author_name'] + ) + + ( + tag_email, + ind1_email, + ind2_email, + subfield_email + ) = self._get_marc_field_parts( + CFG_BIBSWORD_MARC_FIELDS['author_email'] + ) + + ( + tag_aff, + ind1_aff, + ind2_aff, + subfield_aff + ) = self._get_marc_field_parts( + CFG_BIBSWORD_MARC_FIELDS['author_affiliation'] + ) + + for record_author_name in self._record.get(tag_name, []): + if _decapitalize(record_author_name[1:3]) == ( + ind1_name, ind2_name + ): + for (code, value) in record_author_name[0]: + if code.lower() == subfield_name: + author_name = value + break + if tag_name == tag_email and (ind1_name, ind2_name) == ( + ind1_email, ind2_email + ): + for (code, value) in record_author_name[0]: + if code.lower() == subfield_email: + author_email = value + break + if tag_name == tag_aff and (ind1_name, ind2_name) == ( + ind1_aff, ind2_aff + ): + for (code, value) in record_author_name[0]: + if code.lower() == subfield_aff: + author_affiliation = value + break + # Only get the first occurrence + break - metadata['contributors'] = contributors + if tag_name != tag_email or (ind1_name, ind2_name) != ( + ind1_email, ind2_email + ): + for record_author_email in self._record.get(tag_email, []): + if _decapitalize(record_author_email[1:3]) == ( + ind1_email, ind2_email + ): + for (code, value) in record_author_email[0]: + if code.lower() == subfield_email: + author_email = value + break + # Only get the first occurrence + break + + if tag_name != tag_aff or (ind1_name, ind2_name) != ( + ind1_aff, ind2_aff + ): + for record_author_affiliation in self._record.get(tag_aff, []): + if _decapitalize(record_author_affiliation[1:3]) == ( + ind1_aff, ind2_aff + ): + for (code, value) in record_author_affiliation[0]: + if code.lower() == subfield_aff: + author_affiliation = value + break + # Only get the first occurrence + break + + self._author = (author_name, author_email, author_affiliation) + + return self._author + + def set_record_author( + self, + (author_name, author_email, author_affiliation) + ): + self._author = (author_name, author_email, author_affiliation) + + def _equalize_lists(self, *args): + + max_len = max([len(arg) for arg in args]) + + for arg in args: + diff = max_len - len(arg) + if diff > 0: + arg.extend([None] * diff) + + def get_record_contributors(self, force=False): + + if '_contributors' not in self.__dict__.keys() or force: + + self._contributors = [] + + contributors_names = [] + contributors_emails = [] + contributors_affiliations = [] + + ( + tag_name, + ind1_name, + ind2_name, + subfield_name + ) = self._get_marc_field_parts( + CFG_BIBSWORD_MARC_FIELDS['contributor_name'] + ) + + ( + tag_aff, + ind1_aff, + ind2_aff, + subfield_aff + ) = self._get_marc_field_parts( + CFG_BIBSWORD_MARC_FIELDS['contributor_affiliation'] + ) + + for record_contributor_name in self._record.get(tag_name, []): + contributor_name = None + contributor_email = None + contributor_affiliation = None + if _decapitalize(record_contributor_name[1:3]) == ( + ind1_name, ind2_name + ): + for (code, value) in record_contributor_name[0]: + if code.lower() == subfield_name: + contributor_name = value + break + contributors_names.append(contributor_name) + if tag_name == tag_aff and (ind1_name, ind2_name) == ( + ind1_aff, ind2_aff + ): + for (code, value) in record_contributor_name[0]: + if code.lower() == subfield_aff: + contributor_affiliation = value + break + contributors_affiliations.append( + contributor_affiliation + ) + + if tag_name != tag_aff or (ind1_name, ind2_name) != ( + ind1_aff, ind2_aff + ): + for record_contributor_affiliation in self._record.get( + tag_aff, [] + ): + if _decapitalize( + record_contributor_affiliation[1:3] + ) == (ind1_aff, ind2_aff): + for ( + code, + value + ) in record_contributor_affiliation[0]: + if code.lower() == subfield_aff: + contributor_affiliation = value + break + contributors_affiliations.append( + contributor_affiliation + ) + + self._equalize_lists( + contributors_names, + contributors_emails, + contributors_affiliations + ) + self._contributors = zip( + contributors_names, + contributors_emails, + contributors_affiliations + ) + + return self._contributors + + def set_record_contributors( + self, + (contributors_names, contributors_emails, contributors_affiliations) + ): + self._contributors = zip( + contributors_names, + contributors_emails, + contributors_affiliations + ) + + def get_record_report_number(self, force=False): + + if '_rn' not in self.__dict__.keys() or force: + + self._rn = "" + + (tag, ind1, ind2, subfield) = self._get_marc_field_parts( + CFG_BIBSWORD_MARC_FIELDS['rn'] + ) + + for record_rn in self._record.get(tag, []): + if _decapitalize(record_rn[1:3]) == (ind1, ind2): + for (code, value) in record_rn[0]: + if code.lower() == subfield: + self._rn = value + break + # Only get the first occurrence + break + + return self._rn + + def set_record_report_number(self, rn): + self._rn = rn + + def get_record_additional_report_numbers(self, force=False): + + if '_additional_rn' not in self.__dict__.keys() or force: + + self._additional_rn = [] + + (tag, ind1, ind2, subfield) = self._get_marc_field_parts( + CFG_BIBSWORD_MARC_FIELDS['additional_rn'] + ) + + for record_additional_rn in self._record.get(tag, []): + if _decapitalize(record_additional_rn[1:3]) == (ind1, ind2): + for (code, value) in record_additional_rn[0]: + if code.lower() == subfield: + self._additional_rn.append(value) + break + + self._additional_rn = tuple(self._additional_rn) + + return self._additional_rn + + def set_record_additional_report_numbers(self, additional_rn): + self._additional_rn = tuple(additional_rn) + + def get_record_doi(self, force=False): + + if '_doi' not in self.__dict__.keys() or force: + + self._doi = "" + + (tag, ind1, ind2, subfield) = self._get_marc_field_parts( + CFG_BIBSWORD_MARC_FIELDS['doi'] + ) + + for record_doi in self._record.get(tag, []): + if _decapitalize(record_doi[1:3]) == (ind1, ind2): + for (code, value) in record_doi[0]: + if code.lower() == subfield: + self._doi = value + break + # Only get the first occurrence + break + + return self._doi + + def set_record_doi(self, doi): + self._doi = doi + + def get_record_comments(self, force=False): + + if '_comments' not in self.__dict__.keys() or force: + + self._comments = [] + + (tag, ind1, ind2, subfield) = self._get_marc_field_parts( + CFG_BIBSWORD_MARC_FIELDS['comments'] + ) + + for record_comments in self._record.get(tag, []): + if _decapitalize(record_comments[1:3]) == (ind1, ind2): + for (code, value) in record_comments[0]: + if code.lower() == subfield: + self._comments.append(value) + break + + self._comments = tuple(self._comments) + + return self._comments + + def set_record_comments(self, comments): + self._comments = tuple(comments) + + def get_record_internal_notes(self, force=False): + + if '_internal_notes' not in self.__dict__.keys() or force: + + self._internal_notes = [] + + (tag, ind1, ind2, subfield) = self._get_marc_field_parts( + CFG_BIBSWORD_MARC_FIELDS['internal_notes'] + ) + + for record_internal_notes in self._record.get(tag, []): + if _decapitalize(record_internal_notes[1:3]) == (ind1, ind2): + for (code, value) in record_internal_notes[0]: + if code.lower() == subfield: + self._internal_notes.append(value) + break + + self._internal_notes = tuple(self._internal_notes) + + return self._internal_notes + + def set_record_internal_notes(self, internal_notes): + self._internal_notes = tuple(internal_notes) + + def get_record_journal_info(self, force=False): + + if '_journal_info' not in self.__dict__.keys() or force: + + journal_code = "" + journal_title = "" + journal_page = "" + journal_year = "" + + ( + tag_code, + ind1_code, + ind2_code, + subfield_code + ) = self._get_marc_field_parts( + CFG_BIBSWORD_MARC_FIELDS['journal_code'] + ) + + ( + tag_title, + ind1_title, + ind2_title, + subfield_title + ) = self._get_marc_field_parts( + CFG_BIBSWORD_MARC_FIELDS['journal_title'] + ) + + ( + tag_page, + ind1_page, + ind2_page, + subfield_page + ) = self._get_marc_field_parts( + CFG_BIBSWORD_MARC_FIELDS['journal_page'] + ) + + ( + tag_year, + ind1_year, + ind2_year, + subfield_year + ) = self._get_marc_field_parts( + CFG_BIBSWORD_MARC_FIELDS['journal_year'] + ) + + for record_journal_code in self._record.get(tag_code, []): + if _decapitalize(record_journal_code[1:3]) == ( + ind1_code, ind2_code + ): + for (code, value) in record_journal_code[0]: + if code.lower() == subfield_code: + journal_code = value + break + if tag_code == tag_title and (ind1_code, ind2_code) == ( + ind1_title, ind2_title + ): + for (code, value) in record_journal_code[0]: + if code.lower() == subfield_title: + journal_title = value + break + if tag_code == tag_page and (ind1_code, ind2_code) == ( + ind1_page, ind2_page + ): + for (code, value) in record_journal_code[0]: + if code.lower() == subfield_page: + journal_page = value + break + if tag_code == tag_year and (ind1_code, ind2_code) == ( + ind1_year, ind2_year + ): + for (code, value) in record_journal_code[0]: + if code.lower() == subfield_year: + journal_year = value + break + # Only get the first occurrence + break + if tag_code != tag_title or (ind1_code, ind2_code) != ( + ind1_title, ind2_title + ): + for record_journal_title in self._record.get(tag_title, []): + if _decapitalize(record_journal_title[1:3]) == ( + ind1_title, ind2_title + ): + for (code, value) in record_journal_title[0]: + if code.lower() == subfield_title: + journal_title = value + break + # Only get the first occurrence + break + + if tag_code != tag_page or (ind1_code, ind2_code) != ( + ind1_page, ind2_page + ): + for record_journal_page in self._record.get(tag_page, []): + if _decapitalize(record_journal_page[1:3]) == ( + ind1_page, ind2_page + ): + for (code, value) in record_journal_page[0]: + if code.lower() == subfield_page: + journal_page = value + break + # Only get the first occurrence + break + + if tag_code != tag_year or (ind1_code, ind2_code) != ( + ind1_year, ind2_year + ): + for record_journal_year in self._record.get(tag_year, []): + if _decapitalize(record_journal_year[1:3]) == ( + ind1_year, ind2_year + ): + for (code, value) in record_journal_year[0]: + if code.lower() == subfield_year: + journal_year = value + break + # Only get the first occurrence + break + + self._journal_info = ( + journal_code, + journal_title, + journal_page, + journal_year + ) + + return self._journal_info + + def set_record_journal_info( + self, + (journal_code, journal_title, journal_page, journal_year) + ): + self._journal_info = ( + journal_code, + journal_title, + journal_page, + journal_year + ) + + def get_record_modification_date(self, force=False): + if '_modification_date' not in self.__dict__.keys() or force: + self._modification_date = get_modification_date( + self.get_recid(), + CFG_BIBSWORD_UPDATED_DATE_MYSQL_FORMAT + + CFG_BIBSWORD_LOCAL_TIMEZONE + ) + return self._modification_date + + def set_record_modification_date(self, modification_date): + self._modification_date = modification_date + + def get_accepted_file_types(self): + if '_accepted_file_types' not in self.__dict__.keys(): + if '_collection_url' not in self.__dict__.keys(): + self._collection_url = self.get_collection_url() + self._accepted_file_types = self._server.get_accepted_file_types( + self._collection_url + ) + return self._accepted_file_types + + def get_maximum_file_size(self): + """ + In bytes. + """ + if '_maximum_file_size' not in self.__dict__.keys(): + self._maximum_file_size = self._server.get_maximum_file_size() + return self._maximum_file_size + + def get_record_files(self): + + if '_files' not in self.__dict__.keys(): + + # Fetch all the record's files and cross-check the list + # against the accepted file types of the chosen collection + # and the maximum file size + + accepted_file_types = self.get_accepted_file_types() + maximum_file_size = self.get_maximum_file_size() + + self._files = {} + counter = 0 + + brd = BibRecDocs(self._recid) + bdfs = brd.list_latest_files() + for bdf in bdfs: + extension = bdf.get_superformat() + checksum = bdf.get_checksum() + path = bdf.get_full_path() + url = bdf.get_url() + name = bdf.get_full_name() + size = bdf.get_size() + mime = bdf.mime + + if (extension in accepted_file_types) and ( + size <= maximum_file_size + ): + counter += 1 + self._files[counter] = { + 'name': name, + 'path': path, + 'url': url, + 'checksum': checksum, + 'size': size, + 'mime': mime, + } + + return self._files + + def get_files_indexes(self): + if '_files_indexes' not in self.__dict__.keys(): + self._files_indexes = () + return self._files_indexes + + def set_files_indexes(self, files_indexes): + self._files_indexes = tuple(map(int, files_indexes)) + + def get_server_id(self): + if '_server_id' not in self.__dict__.keys(): + self._server_id = None + return self._server_id + + def set_server_id(self, server_id): + self._server_id = server_id + + def get_server(self): + if '_server' not in self.__dict__.keys(): + self._server = None + return self._server + + def set_server(self, server): + self._server = server + + def get_available_collections(self): + if '_available_collections' not in self.__dict__.keys(): + if '_server' not in self.__dict__.keys(): + self._server = self.get_server() + self._available_collections = self._server.get_collections() + return self._available_collections + + def get_collection_url(self): + if '_collection_url' not in self.__dict__.keys(): + self._collection_url = None + return self._collection_url + + def set_collection_url(self, collection_url): + self._collection_url = collection_url + + def get_available_categories(self): + if '_available_categories' not in self.__dict__.keys(): + if '_collection_url' not in self.__dict__.keys(): + self._collection_url = self.get_collection_url() + self._available_categories = self._server.get_categories( + self._collection_url + ) + return self._available_categories + + def set_mandatory_category_url(self, mandatory_category_url): + self._mandatory_category_url = mandatory_category_url + + def get_mandatory_category_url(self): + if '_mandatory_category_url' not in self.__dict__.keys(): + self._mandatory_category_url = None + return self._mandatory_category_url + + def set_optional_categories_urls(self, optional_categories_urls): + self._optional_categories_urls = tuple(optional_categories_urls) + + def get_optional_categories_urls(self): + if '_optional_categories_urls' not in self.__dict__.keys(): + self._optional_categories_urls = () + return self._optional_categories_urls + + def submit(self): + + # Prepare the URL + url = self.get_collection_url() + + # Prepare the metadata + + # Prepare the mandatory and optional categories + available_categories = self.get_available_categories() + mandatory_category_url = self.get_mandatory_category_url() + optional_categories_urls = self.get_optional_categories_urls() + mandatory_category = { + "term": mandatory_category_url, + "scheme": available_categories[ + "mandatory" + ][ + mandatory_category_url + ][ + "scheme" + ], + "label": available_categories[ + "mandatory" + ][ + mandatory_category_url + ][ + "label" + ], + } + optional_categories = [] + for optional_category_url in optional_categories_urls: + optional_categories.append( + { + "term": optional_category_url, + "scheme": available_categories[ + "optional" + ][ + optional_category_url + ][ + "scheme" + ], + "label": available_categories[ + "optional" + ][ + optional_category_url + ][ + "label" + ], + } + ) + optional_categories = tuple(optional_categories) + + # Get all the metadata together + metadata = { + "abstract": self.get_record_abstract(), + "additional_rn": self.get_record_additional_report_numbers(), + "author": self.get_record_author(), + "comments": self.get_record_comments(), + "contributors": self.get_record_contributors(), + "doi": self.get_record_doi(), + "internal_notes": self.get_record_internal_notes(), + "journal_info": self.get_record_journal_info(), + "rn": self.get_record_report_number(), + "title": self.get_record_title(), + "modification_date": self.get_record_modification_date(), + "recid": self.get_recid(), + "mandatory_category": mandatory_category, + "optional_categories": optional_categories, + } + + # Prepare the media + media = {} + files = self.get_record_files() + files_indexes = self.get_files_indexes() + for file_index in files_indexes: + media[file_index] = files[file_index] + + # Perform the submission and save the result + self._result = self._server.submit( + metadata, + media, + url + ) + + # Return the result + return self._result + + def get_result(self): + if '_result' not in self.__dict__.keys(): + self._result = None + return self._result + + +def perform_submit( + uid, + record_id, + server_id, + ln +): + """ + """ - #--------------------------------------------------------------------------- - # get the summary of the document (mandatory) - #--------------------------------------------------------------------------- + if record_id and record_exists(record_id): + + # At this point we have a valid recid, go ahead and create the + # submission object. + submission = BibSwordSubmission( + record_id, + uid + ) + + # Gather useful information about the submission. + title = submission.get_record_title() + author = submission.get_record_author()[0] + report_number = submission.get_record_report_number() + + # Get the submission ID. + sid = submission.get_sid() + + # Store the submission. + stored_submission_successfully_p = _store_temp_submission( + sid, + submission + ) + + # Inform the user and administrators in case of problems and exit. + if not stored_submission_successfully_p: + msg = ("BibSword: Unable to store the submission in the DB " + + "(sid={0}, uid={1}, record_id={2}, server_id={3}).").format( + sid, + uid, + record_id, + server_id + ) + raise_exception( + msg=msg, + alert_admin=True + ) + _ = gettext_set_language(ln) + out = _("An error has occured. " + + "The administrators have been informed.") + return out + + # Get the list of available servers. + servers = sword_client_db.get_servers() + # Only keep the server ID and name. + if servers: + servers = [server[:2] for server in servers] + + # If the server_id has already been chosen, + # then move to the next step automatically. + step = 0 + if server_id: + step = 1 + + # Create the basic starting interface for this submission. + out = sword_client_template.tmpl_submit( + sid, + record_id, + title, + author, + report_number, + step, + servers, + server_id, + ln + ) + + return out - if 'summary' not in metadata: - if 'summary' not in metadata and \ - not metadata_from_marcxml['summary']: - metadata['error'].append('No summary given !') - metadata['summary'] = "" - else: - metadata['summary'] = metadata_from_marcxml['summary'] else: - if metadata['summary'] == '': - metadata['error'].append( - 'No summary given !') - - - #--------------------------------------------------------------------------- - # get the url and the label of the categories for the document (mandatory) - #--------------------------------------------------------------------------- - - if 'categories' not in metadata: - metadata['categories'] = [] - - - #--------------------------------------------------------------------------- - # get the report number of the document (optional) - #--------------------------------------------------------------------------- - - if 'report_nos' not in metadata: - metadata['report_nos'] = [] - if 'report_nos' in metadata_from_marcxml: - for report_no in metadata_from_marcxml['report_nos']: - if report_no != '': - metadata['report_nos'].append(report_no) - - if metadata.get('id_record') == '' and len(metadata['report_nos']) > 0: - metadata['id_record'] = metadata['report_nos'][0] - - - #--------------------------------------------------------------------------- - # get the journal references of the document (optional) - #--------------------------------------------------------------------------- - - if 'journal_refs' not in metadata: - metadata['journal_refs'] = [] - if 'journal_refs' in metadata_from_marcxml: - for journal_ref in metadata_from_marcxml['journal_refs']: - if journal_ref != '': - metadata['journal_refs'].append(journal_ref) - - - #--------------------------------------------------------------------------- - # get the doi of the document (optional) - #--------------------------------------------------------------------------- + # This means that no record_id was given, inform the user that a + # record_id is necessary for the submission to start. + _ = gettext_set_language(ln) + out = _("Unable to submit invalid record. " + + "Please submit a valid record and try again.") + return out + + +def perform_submit_step_1( + sid, + server_id, + ln +): + """ + """ - if 'doi' not in metadata: - if 'doi' not in metadata_from_marcxml: - metadata['doi'] = "" - else: - metadata['doi'] = metadata_from_marcxml['doi'] + _ = gettext_set_language(ln) + + submission = _retrieve_temp_submission(sid) + if submission is None: + msg = ("BibSword: Unable to retrieve the submission from the DB " + + "(sid={0}, server_id={1}).").format( + sid, + server_id + ) + raise_exception( + msg=msg, + alert_admin=True + ) + _ = gettext_set_language(ln) + out = _("An error has occured. " + + "The administrators have been informed.") + return out + + record_id = submission.get_recid() + is_submission_archived = sword_client_db.is_submission_archived( + record_id, + server_id + ) + if is_submission_archived: + out = _("This record has already been submitted to this server." + + "
    Please start again and select a different server.") + return out + + server_settings = sword_client_db.get_servers( + server_id=server_id, + with_dict=True + ) + if not server_settings: + msg = ("BibSword: Unable to find the server settings in the DB " + + "(sid={0}, server_id={1}).").format( + sid, + server_id + ) + raise_exception( + msg=msg, + alert_admin=True + ) + _ = gettext_set_language(ln) + out = _("An error has occured. " + + "The administrators have been informed.") + return out + + server_settings = server_settings[0] + server_object = _initialize_server(server_settings) + if not server_object: + msg = ("BibSword: Unable to initialize the server " + + "(sid={0}, server_id={1}).").format( + sid, + server_id + ) + raise_exception( + msg=msg, + alert_admin=True + ) + _ = gettext_set_language(ln) + out = _("An error has occured. " + + "The administrators have been informed.") + return out + + # Update the server's service document if needed. + if server_object.needs_to_be_updated(): + server_object.update() + + submission.set_server_id(server_id) + submission.set_server(server_object) + + # Get the available collections + available_collections = submission.get_available_collections() + + # Prepare the collections for the HTML select element + collections = sorted( + [ + ( + available_collection, + available_collections[available_collection]["title"] + ) for available_collection in available_collections + ], + key=lambda available_collection: available_collection[1] + ) + + # Update the stored submission. + updated_submission_successfully_p = _update_temp_submission( + sid, + submission + ) + if not updated_submission_successfully_p: + msg = ("BibSword: Unable to update the submission in the DB " + + "(sid={0}, server_id={1}).").format( + sid, + server_id + ) + raise_exception( + msg=msg, + alert_admin=True + ) + _ = gettext_set_language(ln) + out = _("An error has occured. " + + "The administrators have been informed.") + return out + + out = sword_client_template.submit_step_2_details( + collections, + ln + ) + + return out + + +def perform_submit_step_2( + sid, + collection_url, + ln +): + """ + """ + _ = gettext_set_language(ln) + + submission = _retrieve_temp_submission(sid) + if submission is None: + msg = ("BibSword: Unable to retrieve the submission from the DB " + + "(sid={0}).").format( + sid + ) + raise_exception( + msg=msg, + alert_admin=True + ) + _ = gettext_set_language(ln) + out = _("An error has occured. " + + "The administrators have been informed.") + return out + + submission.set_collection_url(collection_url) + + # Get the available categories + available_categories = submission.get_available_categories() + + # Prepare the categories for the HTML select element + mandatory_categories = sorted( + [ + ( + available_mandatory_category, + available_categories[ + "mandatory" + ][ + available_mandatory_category + ][ + "label" + ] + ) for available_mandatory_category in available_categories[ + "mandatory" + ] + ], + key=lambda available_mandatory_category: available_mandatory_category[ + 1 + ] + ) + optional_categories = sorted( + [ + ( + available_optional_category, + available_categories[ + "optional" + ][ + available_optional_category + ][ + "label" + ] + ) for available_optional_category in available_categories[ + "optional" + ] + ], + key=lambda available_optional_category: available_optional_category[ + 1 + ] + ) + + # Update the stored submission. + updated_submission_successfully_p = _update_temp_submission( + sid, + submission + ) + if not updated_submission_successfully_p: + msg = ("BibSword: Unable to update the submission in the DB " + + "(sid={0}).").format( + sid + ) + raise_exception( + msg=msg, + alert_admin=True + ) + _ = gettext_set_language(ln) + out = _("An error has occured. " + + "The administrators have been informed.") + return out + + out = sword_client_template.submit_step_3_details( + mandatory_categories, + optional_categories, + ln + ) + + return out + + +def perform_submit_step_3( + sid, + mandatory_category_url, + optional_categories_urls, + ln +): + """ + """ - #--------------------------------------------------------------------------- - # get the comment of the document (optional) - #--------------------------------------------------------------------------- + _ = gettext_set_language(ln) + + submission = _retrieve_temp_submission(sid) + if submission is None: + msg = ("BibSword: Unable to retrieve the submission from the DB " + + "(sid={0}).").format( + sid + ) + raise_exception( + msg=msg, + alert_admin=True + ) + _ = gettext_set_language(ln) + out = _("An error has occured. " + + "The administrators have been informed.") + return out + + submission.set_mandatory_category_url(mandatory_category_url) + if optional_categories_urls: + submission.set_optional_categories_urls(optional_categories_urls) + + metadata = { + 'title': submission.get_record_title(), + 'abstract': submission.get_record_abstract(), + 'author': submission.get_record_author(), + 'contributors': submission.get_record_contributors(), + 'rn': submission.get_record_report_number(), + 'additional_rn': submission.get_record_additional_report_numbers(), + 'doi': submission.get_record_doi(), + 'comments': submission.get_record_comments(), + 'internal_notes': submission.get_record_internal_notes(), + 'journal_info': submission.get_record_journal_info(), + } + + files = submission.get_record_files() + + # Update the stored submission. + updated_submission_successfully_p = _update_temp_submission( + sid, + submission + ) + if not updated_submission_successfully_p: + msg = ("BibSword: Unable to update the submission in the DB " + + "(sid={0}).").format( + sid + ) + raise_exception( + msg=msg, + alert_admin=True + ) + _ = gettext_set_language(ln) + out = _("An error has occured. " + + "The administrators have been informed.") + return out + + out = sword_client_template.submit_step_4_details( + metadata, + files, + CFG_BIBSWORD_MAXIMUM_NUMBER_OF_CONTRIBUTORS, + ln + ) + + return out + + +def perform_submit_step_4( + sid, + ( + rn, + additional_rn, + title, + author_fullname, + author_email, + author_affiliation, + abstract, + contributor_fullname, + contributor_email, + contributor_affiliation, + files_indexes + ), + ln +): + """ + """ - if 'comment' not in metadata: - if 'comment' not in metadata_from_marcxml: - metadata['comment'] = "" + _ = gettext_set_language(ln) + + submission = _retrieve_temp_submission(sid) + if submission is None: + msg = ("BibSword: Unable to retrieve the submission from the DB " + + "(sid={0}).").format( + sid + ) + raise_exception( + msg=msg, + alert_admin=True + ) + _ = gettext_set_language(ln) + out = _("An error has occured. " + + "The administrators have been informed.") + return out + + # Set the updated metadata + submission.set_record_title(title) + submission.set_record_abstract(abstract) + submission.set_record_author( + (author_fullname, author_email, author_affiliation) + ) + submission.set_record_contributors( + (contributor_fullname, contributor_email, contributor_affiliation) + ) + submission.set_record_report_number(rn) + submission.set_record_additional_report_numbers(additional_rn) + # submission.set_record_doi(doi) + # submission.set_record_comments(comments) + # submission.set_record_internal_notes(internal_notes) + # submission.set_record_journal_info( + # (journal_code, journal_title, journal_page, journal_year) + # ) + + # Set the chosen files indexes + submission.set_files_indexes(files_indexes) + + # Perform the submission + result = submission.submit() + + if result is None or not result or result.get('error', False): + # Update the stored submission + updated_submission_successfully_p = _update_temp_submission( + sid, + submission + ) + + if not updated_submission_successfully_p: + msg = ("BibSword: Unable to update the submission in the DB " + + "(sid={0}).").format( + sid + ) + raise_exception( + msg=msg, + alert_admin=True + ) + _ = gettext_set_language(ln) + out = _("An error has occured. " + + "The administrators have been informed.") + return out + + # TODO: Should we give the user detailed feedback on the error + # in the result (if it exsists?), instead of just notifying + # the admins? + msg = ("BibSword: The submission failed " + + "(sid={0}, result={1}).").format( + sid, + str(result) + ) + raise_exception( + msg=msg, + alert_admin=True + ) + + _ = gettext_set_language(ln) + if result.get("error", False): + out = _("The submission could not be completed successfully " + + "and the following error has been reported:" + + "
    {0}
    " + + "The administrators have been informed.").format( + result.get("msg", _("Uknown error.")) + ) else: - metadata['comment'] = metadata_from_marcxml['comment'] - - return metadata - - -def submit_metadata(server_id, deposit_url, metadata, username= '', email=''): - ''' - Submit the given metadata xml entry to the deposit_url. A username and - an email address maight be used to proced on behalf of the real author - of the document - @param metadata: xml atom entry containing every metadata and links - @param deposit_url: url of the deposition (usually a collection' url) - @param username: name of the user depositing 'on behalf of' an author - @param email: allow user to get an acknowledgement of the deposit - @return: xml atom entry containing submission acknowledgement or error - ''' + out = _("An error has occured. " + + "The administrators have been informed.") + return out - if username != '' and email != '': - onbehalf = '''"%s" <%s>''' % (username, email) else: - onbehalf = '' - - authentication_info = get_remote_server_auth(server_id) - connection = RemoteSwordServer(authentication_info) - - tmp_file = open("/tmp/file", "w") - tmp_file.write(deposit_url + '\n' + onbehalf) - - return connection.metadata_submission(deposit_url, metadata, onbehalf) - - -def perform_submission_process(server_id, collection, recid, user_info, - metadata=None, medias=None, marcxml=""): - ''' - This function is an abstraction of the 2 steps submission process. It - submit the media to a collection, format the metadata and submit them - to the same collection. In case of error in one of the 3 operations, it - stops the process and send an error message back. In addition, this - function insert informations in the swrCLIENTDATA and MARC to avoid sending a - record twice in the same remote server - @param server_id: remote server id on the swrREMOTESERVER table - @param user_info: invenio user infos of the submitter - @param metadata: dictionnary containing some informations - @param collection: url of the place where to deposit the record - @param marcxml: place where to find important information to the record - @param recid: id of the record that can be found if no marcxml - return: tuple containing deposit informations and submission informations - ''' - - if metadata == None: - metadata = {} - - if medias == None: - medias = [] - - # dictionnary containing 2 steps response and possible errors - response = {'error':'', - 'message':'', - 'deposit_media':'', - 'submit_metadata':'', - 'row_id': ''} - - # get the marcxml file (if needed) - if marcxml == '': - if recid == '': - response['error'] = 'You must give a marcxml file or a record id' - return response - marcxml = get_marcxml_from_record(recid) - - - #*************************************************************************** - # Check if record was already submitted - #*************************************************************************** - - # get the record id in the marcxml file - record_id = '' - record_id = get_report_number_from_macrxml(marcxml) - if record_id == '': - response['error'] = 'The marcxml file has no record_id' - return response - - # check if record already sent to the server - if(is_record_sent_to_server(server_id, recid) == True): - response['error'] = \ - 'The record was already sent to the specified server' - return response - - - #*************************************************************************** - # Get informations for a 'on-behalf-of' submission if needed - #*************************************************************************** - - username = '' - email = '' - author = format_author_from_marcxml(marcxml) - - if author['name'] == user_info['nickname']: - author['email'] = user_info['email'] - else: - username = author['name'] - email = user_info['email'] - - - #*************************************************************************** - # Get the media from the marcxml (if not already made) - #*************************************************************************** - - - media = get_medias_to_submit(medias) - if media == {}: - response['error'] = 'No media to submit' - return response - - deposit_status = deposit_media(server_id, media, collection, username, - email) - - # check if any answer was given - if deposit_status == '': - response['error'] = 'Error during media deposit process' - return response - - tmpfd = NamedTemporaryFile(mode='w', suffix='.xml', prefix='bibsword_media_', - dir=CFG_TMPDIR, delete=False) - tmpfd.write(deposit_status) - tmpfd.close() - - - #*************************************************************************** - # format the metadata files - #*************************************************************************** - - metadata = format_metadata(marcxml, deposit_status, user_info, - metadata) - - arxiv = ArXivFormat() - metadata_atom = arxiv.format_metadata(metadata) - - - #*************************************************************************** - # submit the metadata - #*************************************************************************** - - tmpfd = NamedTemporaryFile(mode='w', suffix='.xml', prefix='bibsword_metadata_', - dir=CFG_TMPDIR, delete=False) - tmpfd.write(metadata_atom) - tmpfd.close() - - submit_status = submit_metadata(server_id, collection, metadata_atom, - username, email) - - tmpfd = NamedTemporaryFile(mode='w', suffix='.xml', prefix='bibsword_submit_', - dir=CFG_TMPDIR, delete=False) - tmpfd.write(submit_status) - tmpfd.close() - - # check if any answer was given - if submit_status == '': - response['message'] = '' - response['error'] = 'Problem during submission process' - return response - - - #*************************************************************************** - # Parse the submit result - #*************************************************************************** - - # get the submission's remote id from the response - remote_id = format_id_from_submission(submit_status) - response['remote_id'] = remote_id - - #get links to medias, metadata and status - links = format_links_from_submission(submit_status) - response['links'] = links - - #insert the submission in the swrCLIENTDATA entry - row_id = insert_into_swr_clientdata(server_id, - recid, - metadata['id_record'], - remote_id, - user_info['id'], - user_info['nickname'], - user_info['email'], - deposit_status, - submit_status, - links['media'], - links['metadata'], - links['status']) - - #insert information field in the marc file - current_date = time.strftime("%Y-%m-%d %H:%M:%S") - update_marcxml_with_info(recid, user_info['nickname'], current_date, - remote_id) - - # format and return the response - response['submit_metadata'] = submit_status - response['row_id'] = row_id - - return response - - -def get_servicedocument(id_server): - ''' - This metode get the xml service document file discribing the collections - and the categories of a remote server. If the servicedocument is saved - in the swrREMOTESERVER table or if it has not been load since a certain - time, it is dynamically loaded from the SWORD remote server. - @param id_server: id of the server where to get the servicedocument - @return: service document in a String - ''' - - last_update = get_last_update(id_server) - - time_machine = datetime.datetime.now() - time_now = int(time.mktime(time_machine.timetuple())) - - delta_time = time_now - int(last_update) - - service = select_servicedocument(id_server) - - update = 0 - if delta_time > CFG_BIBSWORD_SERVICEDOCUMENT_UPDATE_TIME: - update = 1 - elif service == '': - update = 1 - - if update == 1: - authentication_info = get_remote_server_auth(id_server) - connection = RemoteSwordServer(authentication_info) - service = connection.get_remote_collection(\ - authentication_info['url_servicedocument']) - if service == '': - service = select_servicedocument(id_server) - else: - update_servicedocument(service, id_server) - - return service - - -#------------------------------------------------------------------------------- -# Implementation of the Command line client -#------------------------------------------------------------------------------- - -def usage(exitcode=1, msg=""): - """Prints usage info.""" - if msg: - sys.stderr.write("*************************************************"\ - "***********************************\n") - sys.stderr.write(" ERROR\n") - sys.stderr.write("message: %s \n" % msg) - sys.stderr.write("*************************************************"\ - "***********************************\n") - sys.stderr.write("\n") - sys.stderr.write("Usage: %s [options] \n" % sys.argv[0]) - sys.stderr.write("\n") - sys.stderr.write("*****************************************************"\ - "**********************************\n") - sys.stderr.write(" OPTIONS\n") - sys.stderr.write("*****************************************************"\ - "**********************************\n") - sys.stderr.write("-h, --help : Print this help.\n") - sys.stderr.write("-s, --simulation: Proceed in a simulation mode\n") - sys.stderr.write("\n") - sys.stderr.write("*****************************************************"\ - "**********************************\n") - sys.stderr.write(" HELPERS\n") - sys.stderr.write("*****************************************************"\ - "**********************************\n") - sys.stderr.write("-r, --list-remote-servers: List all available remote"\ - " server\n") - sys.stderr.write("-i, --list-server-info --server-id: Display SWORD"\ - " informations about the server \n") - sys.stderr.write("-c, --list-collections --server-id: List collections "\ - "for the specified server\n") - sys.stderr.write("-n, --list-collection-info --server-id --collection_id:"\ - " Display infos about collection \n") - sys.stderr.write("-p, --list-primary-categories --server-id "\ - "--colleciton_id: List mandated categories\n") - sys.stderr.write("-o, --list-optional-categories --server-id "\ - "--collection_id: List secondary categories\n") - sys.stderr.write("-v, --list-submission [--server-id --id_record]: "\ - "List submission entry in swrCLIENTDATA\n") - sys.stderr.write("\n") - sys.stderr.write("*****************************************************"\ - "**********************************\n") - sys.stderr.write(" OERATIONS\n") - sys.stderr.write("*****************************************************"\ - "**********************************\n") - sys.stderr.write("-m, --get-marcxml-from-recid --recid: Display the"\ - " MARCXML file for the given record\n") - sys.stderr.write("-e, --get-media-resource [--marcxml-file|--recid]: "\ - "Display type and url of the media\n") - sys.stderr.write("-z, --compress-media-file [--marcxml-file|--recid]: "\ - "Dipsplay the zipped size archive\n") - sys.stderr.write("-d, --deposit-media --server-id --collection_id "\ - "--media: deposit media in colleciton\n") - sys.stderr.write("-f, --format-metadata --server-id --metadata "\ - "--marcxml: format metadata for the server\n") - sys.stderr.write("-l, --submit-metadata --server-id --collection-id "\ - "--metadata: submit metadata to server\n") - sys.stderr.write("-a, --proceed-submission --server-id --recid "\ - "--metadata: do the entire deposit process\n") - sys.stderr.write("\n") - sys.exit(exitcode) - - -def main(): + # Archive the submission + archived_submission_successfully_p = _archive_submission(submission) + if not archived_submission_successfully_p: + msg = ("BibSword: Unable to archive the submission in the DB " + + "(sid={0}).").format( + sid + ) + raise_exception( + msg=msg, + alert_admin=True + ) + _ = gettext_set_language(ln) + out = _("An error has occured. " + + "The administrators have been informed.") + return out + + # Delete the previously temporary submission + deleted_submission_successfully_p = _delete_temp_submission(sid) + if not deleted_submission_successfully_p: + msg = ("BibSword: Unable to delete the submission in the DB " + + "(sid={0}).").format( + sid + ) + raise_exception( + msg=msg, + alert_admin=True + ) + _ = gettext_set_language(ln) + out = _("An error has occured. " + + "The administrators have been informed.") + return out + + out = sword_client_template.submit_step_final_details(ln) + + return out + + +def _archive_submission(sobject): """ - main entry point for webdoc via command line + Archives the current submission. """ - options = {'action':'', - 'server-id':0, - 'recid':0, - 'collection-id':0, - 'mode':2, - 'marcxml-file': '', - 'collection_url':'', - 'deposit-result':'', - 'metadata':'', - 'proceed-submission':'', - 'list-submission':''} - - try: - opts, args = getopt.getopt(sys.argv[1:], - "hsricnpovmezdfla", - ["help", - "simulation", - "list-remote-servers", - "list-server-info", - "list-collections", - "list-collection-info", - "list-primary-categories", - "list-optional-categories", - "list-submission", - "get-marcxml-from-recid", - "get-media-resource", - "compress-media-file", - "deposit-media", - "format-metadata", - "submit-metadata", - "proceed-submission", - "server-id=", - "collection-id=", - "recid=", - "marcxml-file=", - "collection_url=", - "deposit-result=", - "metadata=", - "yes-i-know" - ]) - - except getopt.GetoptError, err: - usage(1, err) - - if len(opts) == 0: - usage(1, 'No options given') - - if not '--yes-i-know' in sys.argv[1:]: - print "This is an experimental tool. It is disabled for the moment." - sys.exit(0) - - try: - for opt in opts: - if opt[0] in ["-h", "--help"]: - usage(0) - - elif opt[0] in ["-s", "--simulation"]: - options["simulation"] = int(opt[1]) - - #------------------------------------------------------------------- - - elif opt[0] in ["-r", "--list-remote-servers"]: - options["action"] = "list-remote-servers" - - elif opt[0] in ["-i", "--list-server-info"]: - options["action"] = "list-server-info" - - elif opt[0] in ["-c", "--list-collections"]: - options["action"] = "list-collections" - - elif opt[0] in ["-n", "--list-collection-info"]: - options["action"] = "list-collection-info" - - elif opt[0] in ["-p", "--list-primary-categories"]: - options["action"] = "list-primary-categories" - - elif opt[0] in ["-o", "--list-optional-categories"]: - options["action"] = "list-optional-categories" - - elif opt[0] in ["-v", "--list-submission"]: - options['action'] = 'list-submission' - - elif opt[0] in ["-m", "--get-marcxml-from-recid"]: - options["action"] = "get-marcxml-from-recid" - - elif opt[0] in ["-e", "--get-media-resource"]: - options['action'] = "get-media-resource" - - elif opt[0] in ["-z", "--compress-media-file"]: - options['action'] = "compress-media-file" - - elif opt[0] in ["-d", "--deposit-media"]: - options['action'] = "deposit-media" - - elif opt[0] in ["-f", "--format-metadata"]: - options['action'] = "format-metadata" - - elif opt[0] in ["l", "--submit-metadata"]: - options['action'] = "submit-metadata" - elif opt[0] in ["-a", "--proceed-submission"]: - options['action'] = 'proceed-submission' - - - #------------------------------------------------------------------- - - elif opt[0] in ["--server-id"]: - options["server-id"] = int(opt[1]) - - elif opt[0] in ["--collection-id"]: - options["collection-id"] = int(opt[1]) - - elif opt[0] in ["--recid"]: - options["recid"] = int(opt[1]) - - elif opt[0] in ['--marcxml-file']: - options['marcxml-file'] = opt[1] - - elif opt[0] in ['--collection_url']: - options['collection_url'] = opt[1] - - elif opt[0] in ['--deposit-result']: - options['deposit-result'] = opt[1] - - elif opt[0] in ['--metadata']: - options['metadata'] = opt[1] - - - except StandardError, message: - usage(message) - - #--------------------------------------------------------------------------- - # --check parameters type - #--------------------------------------------------------------------------- - - try: - options["server-id"] = int(options["server-id"]) - except ValueError: - usage(1, "--server-id must be an integer") - - try: - options["collection-id"] = int(options["collection-id"]) - except ValueError: - usage(1, "--collection-id must be an integer") - - try: - options["recid"] = int(options["recid"]) - except ValueError: - usage(1, "--recid must be an integer") - - - #--------------------------------------------------------------------------- - # --list-remote-servers - #--------------------------------------------------------------------------- - - if options['action'] == "list-remote-servers": - servers = list_remote_servers() - for server in servers: - print str(server['id']) +': '+ server['name'] + \ - ' ( ' + server['host'] + ' ) ' - - - #--------------------------------------------------------------------------- - # --list-server-info - #--------------------------------------------------------------------------- - - if options['action'] == "list-server-info": - - info = list_server_info(options['server-id']) - - if info == {}: - print 'Error, no infos found !' - else: - print 'SWORD version: ' + info['version'] - print 'Maximal upload size [Kb]: ' + info['maxUploadSize'] - print 'Implements verbose mode: ' + info['verbose'] - print 'Implementes simulation mode: ' + info['noOp'] - - - #--------------------------------------------------------------------------- - # --list-collections - #--------------------------------------------------------------------------- - - if options['action'] == "list-collections": - - collections = list_collections_from_server(str(options["server-id"])) - - if len(collections) == 0: - usage(1, "Wrong server id, try --get-remote-servers") - - for collection in collections: - print collection['id'] +': '+ collection['label'] + ' - ' + \ - collection['url'] - - - #--------------------------------------------------------------------------- - # --list-collection-info - #--------------------------------------------------------------------------- - - if options['action'] == "list-collection-info": - - info = list_collection_informations(str(options['server-id']), - options['collection-id']) - - print 'Accepted media types:' + user_id = sobject.get_uid() + record_id = sobject.get_recid() + server_id = sobject.get_server_id() + result = sobject.get_result() + alternate_url = result['msg']['alternate'] + edit_url = result['msg']['edit'] + return sword_client_db.archive_submission( + user_id, + record_id, + server_id, + alternate_url, + edit_url + ) + + +def _store_temp_submission(sid, sobject): + """ + Uses cPickle to dump the current submission and stores it. + """ - accept_list = info['accept'] - for accept in accept_list: - print '- ' + accept + sobject_blob = cPickle.dumps(sobject) + return sword_client_db.store_temp_submission(sid, sobject_blob) - print 'collection policy: ' + info['collectionPolicy'] - print 'mediation allowed: ' + info['mediation'] - print 'treatment mode: ' + info['treatment'] - print 'location of accept packaging list: ' + info['acceptPackaging'] - #--------------------------------------------------------------------------- - # --list-primary-categories - #--------------------------------------------------------------------------- +def _retrieve_temp_submission(sid): + """ + Retrieves the current submission and uses cPickle to load it. + """ - if options['action'] == "list-primary-categories": + # call DB function to retrieve the blob + sobject_blob = sword_client_db.retrieve_temp_submission(sid) + sobject = cPickle.loads(sobject_blob) + return sobject - categories = list_mandated_categories(\ - str(options["server-id"]), options["collection-id"]) - if len(categories) == 0: - usage(1, "Wrong server id, try --get-collections") +def _update_temp_submission(sid, sobject): + """ + Uses cPickle to dump the current submission and updates it. + """ - for category in categories: - print category['id'] +': '+ category['label'] + ' - ' + \ - category['url'] + sobject_blob = cPickle.dumps(sobject) + return sword_client_db.update_temp_submission(sid, sobject_blob) - #--------------------------------------------------------------------------- - # --list-optional-categories - #--------------------------------------------------------------------------- +def _delete_temp_submission(sid): + """ + Deletes the given submission. + """ - if options['action'] == "list-optional-categories": + return sword_client_db.delete_temp_submission(sid) - categories = list_optional_categories(\ - str(options["server-id"]), options["collection-id"]) - if len(categories) == 0: - usage(1, "Wrong server id, try --get-collections") +def _decapitalize(field_parts): + return tuple([part.lower() for part in field_parts]) - for category in categories: - print category['id'] +': '+ category['label'] + ' - ' + \ - category['url'] +def perform_request_submissions( + ln +): + """ + Returns the HTML for the Sword client submissions. + """ - #--------------------------------------------------------------------------- - # --list-submission - #--------------------------------------------------------------------------- + submissions = sword_client_db.get_submissions() - if options['action'] == "list-submission": + html = sword_client_template.tmpl_submissions( + submissions, + ln + ) - results = select_submitted_record_infos() + return html - for result in results: - print '\n' - print 'submission id: ' + str(result[0]) - print 'remote server id: ' + str(result[1]) - print 'submitter id: ' + str(result[4]) - print 'local record id: ' + result[2] - print 'remote record id: ' + str(result[3]) - print 'submit date: ' + result[5] - print 'document type: ' + result[6] - print 'media link: ' + result[7] - print 'metadata link: ' + result[8] - print 'status link: ' + result[9] +def perform_request_submission_options( + option, + action, + server_id, + status_url, + ln +): + """ + Perform an action on a given submission based on the selected option + and return the results. + """ - #--------------------------------------------------------------------------- - # --get-marcxml-from-recid - #--------------------------------------------------------------------------- + _ = gettext_set_language(ln) - if options['action'] == "get-marcxml-from-recid": + (error, result) = (None, None) - marcxml = get_marcxml_from_record(options['recid']) + if not option: + error = _("Missing option") - if marcxml == '': - usage(1, "recid %d unknown" % options['recid']) + elif not action: + error = _("Missing action") - else: - print marcxml + else: + if option == "update": - #--------------------------------------------------------------------------- - # --get-media-resource - #--------------------------------------------------------------------------- + if not server_id: + error = _("Missing server ID") - if options['action'] == "get-media-resource": + if not status_url: + error = _("Missing status URL") - if options['marcxml-file'] == '': - if options ['recid'] == 0: - usage (1, "you must provide a metadata file or a valid recid") else: - options['marcxml-file'] = \ - get_marcxml_from_record(options['recid']) - else: - options['marcxml-file'] = open(options['marcxml-file']).read() - - medias = get_media_list(options['recid']) - - for media in medias: - print 'media_link = '+ media['path'] - print 'media_type = '+ media['type'] + if action == "submit": + server_settings = sword_client_db.get_servers( + server_id=server_id, + with_dict=True + ) + if not server_settings: + error = _("The server could not be found") + else: + server_settings = server_settings[0] + server_object = _initialize_server(server_settings) + if not server_object: + error = _("The server could not be initialized") + else: + result = server_object.status(status_url) + if result["error"]: + error = _(result["msg"]) + else: + result = { + "status": ( + result["msg"]["error"] is None + ) and ( + "{0}".format(result["msg"]["status"]) + ) or ( + "{0} ({1})".format( + result["msg"]["status"], + result["msg"]["error"] + ) + ), + "last_updated": _("Just now"), + } + result = json.dumps(result) + else: + error = _("Wrong action") - #--------------------------------------------------------------------------- - # --compress-media-file - #--------------------------------------------------------------------------- - - if options['action'] == "compress-media-file": - - if options['marcxml-file'] != '': - options['media-file-list'] = \ - get_media_list(options['recid']) - elif options ['recid'] != 0: - options['marcxml-file'] = \ - get_marcxml_from_record(options['recid']) - options['media-file-list'] = \ - get_media_list(options['recid']) else: - usage (1, "you must provide a media file list, a metadata file or"+ - " a valid recid") - - print compress_media_file(options['media-file-list']) - - - #--------------------------------------------------------------------------- - # --deposit-media - #--------------------------------------------------------------------------- - - if options['action'] == "deposit-media": - - if options["server-id"] == 0: - usage (1, "You must select a server where to deposit the resource."+ - "\nDo: ./bibSword -l") - - if options['marcxml-file'] != '': - options['media-file-list'] = \ - get_media_list(options['recid']) - elif options ['recid'] != 0: - options['marcxml-file'] = get_marcxml_from_record(options['recid']) - options['media-file-list'] = \ - get_media_list(options['recid']) - else: - usage (1, "you must provide a media file list, a metadata file" + - " or a valid recid") - - collection = 'https://arxiv.org/sword-app/physics-collection' - medias = options['media-file-list'] - server_id = options["server-id"] - - print collection - for media in medias: - print media['type'] - print server_id + error = _("Wrong option") - result = deposit_media(server_id, medias, collection) + return (error, result) - for result in results: - print result +def perform_request_servers( + ln +): + """ + Returns the HTML for the Sword client servers. + """ - #--------------------------------------------------------------------------- - # --format-metadata - #--------------------------------------------------------------------------- - user_info = {'id':'1', - 'nickname':'admin', - 'email': CFG_SITE_ADMIN_EMAIL} - - if options['action'] == "format-metadata": - - if options['marcxml-file'] == '': - if options ['recid'] != 0: - options['marcxml-file'] = \ - get_marcxml_from_record(options['recid']) - else: - usage (1, "you must provide a metadata file or a valid recid") - - deposit = [] - deposit.append(options['deposit-result']) - - print format_metadata(options['marcxml-file'], deposit, - user_info) - - - #--------------------------------------------------------------------------- - # --submit-metadata - #--------------------------------------------------------------------------- - - if options['action'] == "submit-metadata": - - if options['collection_url'] == '': - if options['server-id'] == '' or options['collection-id'] == '': - usage(1, \ - "You must enter a collection or a server-id and a collection-id") - - if options['metadata'] == '': - usage(1, \ - "You must enter the location of the metadata file to submit") - - if options['server-id'] == '': - usage(1, "You must specify the server id") - - metadata = open(options['metadata']).read() - - print submit_metadata(options['server-id'], - options['collection_url'], - metadata, - user_info['nickname'], - user_info['email']) - - - #--------------------------------------------------------------------------- - # --proceed-submission - #--------------------------------------------------------------------------- - - if options['action'] == "proceed-submission": - - if options["server-id"] == 0: - usage (1, "You must select a server where to deposit the resource."+ - "\nDo: ./bibSword -l") - - if options["recid"] == 0: - usage(1, "You must specify the record to submit") - - metadata = {'title':'', - 'id':'', - 'updated':'', - 'author_name':'Invenio Admin', - 'author_email': CFG_SITE_ADMIN_EMAIL, - 'contributors': [], - 'summary':'', - 'categories':[], - 'primary_label':'High Energy Astrophysical Phenomena', - 'primary_url':'http://arxiv.org/terms/arXiv/astro-ph.HE', - 'comment':'', - 'doi':'', - 'report_nos':[], - 'journal_refs':[], - 'links':[]} - - collection = 'https://arxiv.org/sword-app/physics-collection' - - server_id = 1 - - response = perform_submission_process(options["server-id"], user_info, - metadata, collection, '', '', - options['recid']) - - if response['error'] != '': - print 'error: ' + response['error'] - - if response['message'] != '': - print 'message: ' + response['message'] - - for deposit_media in response['deposit_media']: - print 'deposit_media: \n ' + deposit_media - - if response['submit_metadata'] != '': - print 'submit_metadata: \n ' + response['submit_metadata'] - - -#------------------------------------------------------------------------------- -# avoid launching file during inclusion -#------------------------------------------------------------------------------- - -if __name__ == "__main__": - main() - - - -#------------------------------------------------------------------------------- -# Implementation of the Web Client -#------------------------------------------------------------------------------- - -def perform_display_sub_status(first_row=1, offset=10, - action="submitted"): - ''' - Get the given submission status and display it in a html table - @param first_row: first row of the swrCLIENTDATA table to display - @param offset: nb of row to select - @return: html code containing submission status table - ''' - - #declare return values - body = '' - errors = [] - warnings = [] - - if first_row < 1: - first_row = 1 - - submissions = list_submitted_resources(int(first_row)-1, offset, action) + servers = sword_client_db.get_servers() - total_rows = count_nb_submitted_record() - last_row = first_row + offset - 1 - if last_row > total_rows: - last_row = total_rows + html = sword_client_template.tmpl_servers( + servers, + ln + ) - selected_offset = [] - if offset == 5: - selected_offset.append('selected') - else: - selected_offset.append('') + return html - if offset == 10: - selected_offset.append('selected') - else: - selected_offset.append('') - if offset == 25: - selected_offset.append('selected') - else: - selected_offset.append('') +def perform_request_server_options( + option, + action, + server_id, + server, + ln +): + """ + Perform an action on a given server based on the selected option + and return the results. + """ - if offset == 50: - selected_offset.append('selected') - else: - selected_offset.append('') + _ = gettext_set_language(ln) - if offset == total_rows: - selected_offset.append('selected') - else: - selected_offset.append('') + (error, result) = (None, None) - if first_row == 1: - is_first = 'disabled' - else: - is_first = '' + if not option: + error = _("Missing option") - tmp_last = total_rows - offset + elif not action: + error = _("Missing action") - if first_row > tmp_last: - is_last = 'disabled' else: - is_last = '' - - bibsword_template = BibSwordTemplate() - body = bibsword_template.tmpl_display_admin_page(submissions, - first_row, - last_row, - total_rows, - is_first, - is_last, - selected_offset) - - return (body, errors, warnings) + if option == "add": + if action == "prepare": + server = None + available_engines = _CLIENT_SERVERS.keys() + if "__init__" in available_engines: + available_engines.remove("__init__") + result = sword_client_template.tmpl_add_or_modify_server( + server, + available_engines, + ln + ) + elif action == "submit": + if "" in server: + error = _("Insufficient server information") + else: + if not _validate_server(server): + error = _("Wrong server information") + else: + server_id = sword_client_db.add_server(*server) + if server_id: + ( + name, + engine, + username, + password, + email, + update_frequency + ) = server + result = \ + sword_client_template._tmpl_server_table_row( + ( + server_id, + name, + engine, + username, + password, + email, + None, + update_frequency, + None, + ), + ln + ) + else: + error = _("The server could not be added") + else: + error = _("Wrong action") -def perform_display_server_infos(id_server): - ''' - This function get the server infos in the swrREMOTESERVER table - and display it as an html table - @param id_server: id of the server to get the infos - @return: html table code to display - ''' + elif option == "update": - server_infos = select_remote_server_infos(id_server) - bibsword_template = BibSwordTemplate() - return bibsword_template.tmpl_display_remote_server_info(server_infos) + if not server_id: + error = _("Missing server ID") + else: + if action == "submit": + server_settings = sword_client_db.get_servers( + server_id=server_id, + with_dict=True + ) + if not server_settings: + error = _("The server could not be found") + else: + server_settings = server_settings[0] + server_object = _initialize_server(server_settings) + if not server_object: + error = _("The server could not be initialized") + else: + result = server_object.update() + if not result: + error = _("The server could not be updated") + else: + result = _("Just now") + else: + error = _("Wrong action") + + elif option == "modify": + if not server_id: + error = _("Missing server ID") -def perform_display_server_list(error_messages, id_record=""): - ''' - Get the list of remote SWORD server implemented by the BibSword API - and generate the html code that display it as a dropdown list - @param error_messages: list of errors that may happens in validation - @return: string containing the generated html code - ''' + else: + if action == "prepare": + server = sword_client_db.get_servers( + server_id=server_id + ) + if not server: + error = _("The server could not be found") + else: + server = server[0] + available_engines = _CLIENT_SERVERS.keys() + if "__init__" in available_engines: + available_engines.remove("__init__") + result = \ + sword_client_template.tmpl_add_or_modify_server( + server, + available_engines, + ln + ) + elif action == "submit": + if "" in server: + error = _("Insufficient server information") + else: + if not _validate_server(server): + error = _("Wrong server information") + else: + result = sword_client_db.modify_server( + server_id, + *server + ) + if result: + ( + name, + engine, + username, + dummy, + email, + update_frequency + ) = server + result = { + "name": name, + "engine": engine, + "username": username, + "email": email, + "update_frequency": TemplateSwordClient + ._humanize_frequency(update_frequency, ln), + } + result = json.dumps(result) + else: + error = _("The server could not be modified") + else: + error = _("Wrong action") + + elif option == "delete": + if not server_id: + error = _("Missing server ID") - #declare return values - body = '' - errors = [] - warnings = [] + else: + if action == "submit": + result = sword_client_db.delete_servers( + server_id=server_id + ) + if not result: + error = _("The server could not be deleted") + else: + error = _("Wrong action") - # define the list that will contains the remote servers - remote_servers = [] + else: + error = _("Wrong option") - # get the remote servers from the API - remote_servers = list_remote_servers() + return (error, result) - # check that the list contains at least one remote server - if len(remote_servers) == 0: - # add an error to the error list - errors.append('There is no remote server to display') - else: - # format the html body string to containing remote server dropdown list - bibsword_template = BibSwordTemplate() - body = bibsword_template.tmpl_display_remote_servers(remote_servers, - id_record, - error_messages) - - return (body, errors, warnings) - - -def perform_display_collection_list(id_server, id_record, recid, - error_messages=None): - ''' - Get the list of collections contained in the given remote server and - generate the html code that display it as a dropdown list - @param id_server: id of the remote server selected by the user - @param error_messages: list of errors that may happens in validation - @return: string containing the generated html code - ''' - - if error_messages == None: - error_messages = [] - - #declare return values - body = '' - errors = [] - warnings = [] - - # get the server's name and host - remote_servers = list_remote_servers(id_server) - if len(remote_servers) > 0: - remote_server = remote_servers[0] - - # get the server's informations to display - remote_server_infos = list_server_info(id_server) - if remote_server_infos['error'] != '': - error_messages.append(remote_server_infos['error']) - - # get the server's collections - collections = list_collections_from_server(id_server) - - if len(collections) == 0: - # add an error to the error list - error_messages.append('There are no collection to display') - - # format the html body string to containing remote server's dropdown list - bibsword_template = BibSwordTemplate() - body = bibsword_template.tmpl_display_collections(remote_server, - remote_server_infos, - collections, - id_record, - recid, - error_messages) - - return (body, errors, warnings) - - -def perform_display_category_list(id_server, id_collection, id_record, recid, - error_messages=None): - ''' - Get the list of mandated and optional categories contained in the given - collection and generate the html code that display it as a dropdown list - @param id_server: id of the remote server selected by the user - @param id_collection: id of the collection selected by the user - @param error_messages: list of errors that may happens in validation - @return: string containing the generated html code - ''' - - if error_messages == None: - error_messages = [] - - #declare return values - body = '' - errors = [] - warnings = [] - - # get the server's name and host - remote_servers = list_remote_servers(id_server) - if len(remote_servers) > 0: - remote_server = remote_servers[0] - - # get the server's informations to display - remote_server_infos = list_server_info(id_server) - - # get the collection's name and link - collections = list_collections_from_server(id_server) - collection = {} - for item in collections: - if item['id'] == id_collection: - collection = item - - # get the collection's informations to display - collection_infos = list_collection_informations(id_server, id_collection) - - # get primary category list - primary_categories = list_mandated_categories(id_server, id_collection) - - # get optional categories - optional_categories = list_optional_categories(id_server, id_collection) - - # format the html body string to containing category's dropdown list - bibsword_template = BibSwordTemplate() - body = bibsword_template.tmpl_display_categories(remote_server, - remote_server_infos, - collection, - collection_infos, - primary_categories, - optional_categories, - id_record, - recid, - error_messages) - - return (body, errors, warnings) - - -def perform_display_metadata(user, id_server, id_collection, id_primary, - id_categories, id_record, recid, - error_messages=None, metadata=None): - ''' - Get the list of metadata contained in the given marcxml or given by - the users and generate the html code that display it as the summary list - for the submission - @param id_server: id of the remote server selected by the user - @param id_collection: id of the collection selected by the user - @param id_primary: primary collection selected by the user - @param id_record: record number entered by the user - @param recid: record id corresponding to the selected record - @param error_messages: list of errors that may happens in validation - @param metadata: if present, replace the default entry from marcxml - @return: string containing the generated html code - ''' - - if error_messages == None: - error_messages = [] - - if metadata == None: - metadata = {} - - #declare return values - body = '' - errors = [] - warnings = [] - - - # get the server's name and host - remote_servers = list_remote_servers(id_server) - if len(remote_servers) > 0: - remote_server = remote_servers[0] - - - # get the collection's name and link - collections = list_collections_from_server(id_server) - collection = {} - for item in collections: - if item['id'] == id_collection: - collection = item - break - - - # get primary category name and host - primary_categories = list_mandated_categories(id_server, id_collection) - primary = {} - for category in primary_categories: - if category['id'] == id_primary: - primary = category - break - - - categories = [] - if len(id_categories) > 0: - # get optional categories name and host - optional_categories = list_optional_categories(id_server, id_collection) - for item in optional_categories: - category = {} - for id_category in id_categories: - if item['id'] == id_category: - category = item - categories.append(category) - break +def _initialize_server(server_settings): + """ + Initializes and returns the server based on the given settings. + """ - # get the marcxml file - marcxml = get_marcxml_from_record(recid) + server_engine = server_settings.pop('engine') + if server_engine is not None: + server_object = _CLIENT_SERVERS.get(server_engine)(server_settings) + return server_object + return None - # select the medias - if 'selected_medias' in metadata: - medias = get_media_list(recid, metadata['selected_medias']) - else: - medias = get_media_list(recid) - - # get the uploaded media - if 'uploaded_media' in metadata: - if len(metadata['uploaded_media']) > 30: - file_extention = '' - if metadata['type'] == 'application/zip': - file_extention = 'zip' - elif metadata['type'] == 'application/tar': - file_extention = 'tar' - elif metadata['type'] == 'application/docx': - file_extention = 'docx' - elif metadata['type'] == 'application/pdf': - file_extention = 'pdf' - - file_path = '%s/uploaded_file_1.%s' % (CFG_TMPDIR, file_extention) - - # save the file on the tmp directory - tmp_media = open(file_path, 'w') - tmp_media.write(metadata['uploaded_media']) - - media = {'file': metadata['uploaded_media'] , - 'size': str(len(metadata['uploaded_media'])), - 'type': metadata['type'], - 'path': file_path, - 'selected': 'checked="yes"', - 'loaded': True } - medias.append(media) - - if metadata == {}: - # get metadata from marcxml - metadata = format_marcxml_file(marcxml) - - - # format the html body string to containing category's dropdown list - bibsword_template = BibSwordTemplate() - body = bibsword_template.tmpl_display_metadata(user, remote_server, - collection, primary, - categories, medias, - metadata, id_record, - recid, error_messages) - - - return (body, errors, warnings) - - -def perform_submit_record(user, id_server, id_collection, id_primary, - id_categories, recid, metadata=None): - ''' - Get the given informations and submit them to the SWORD remote server - Display the result of the submission or an error message if something - went wrong. - @param user: informations about the submitter - @param id_server: id of the remote server selected by the user - @param id_collection: id of the collection selected by the user - @param id_primary: primary collection selected by the user - @param recid: record id corresponding to the selected record - @param metadata: contains all the metadata to submit - @return: string containing the generated html code - ''' - - if metadata == None: - metadata = {} - - #declare return values - body = '' - errors = [] - warnings = [] - - # get the collection's name and link - collections = list_collections_from_server(id_server) - collection = {} - for item in collections: - if item['id'] == id_collection: - collection = item - - # get primary category name and host - primary_categories = list_mandated_categories(id_server, id_collection) - primary = {} - for category in primary_categories: - if category['id'] == id_primary: - primary = category - metadata['primary_label'] = primary['label'] - metadata['primary_url'] = primary['url'] - break - - # get the secondary categories name and host - categories = [] - if len(id_categories) > 0: - # get optional categories name and host - optional_categories = list_optional_categories(id_server, id_collection) - for item in optional_categories: - category = {} - for id_category in id_categories: - if item['id'] == id_category: - category = item - categories.append(category) - - metadata['categories'] = categories - - # get the marcxml file - marcxml = get_marcxml_from_record(recid) - - user_info = {'id':user['uid'], - 'nickname':user['nickname'], - 'email':user['email']} - - result = perform_submission_process(id_server, collection['url'], recid, - user_info, metadata, metadata['media'], - marcxml) - - body = result - - if result['error'] != '': - body = '

    '+result['error']+'

    ' - else: - submissions = select_submitted_record_infos(0, 1, result['row_id']) - if metadata['filename'] != '': - upload_fulltext(recid, metadata['filename']) - - bibsword_template = BibSwordTemplate() - body = bibsword_template.tmpl_display_list_submission(submissions) +def _validate_server(server_settings): + """ + Returs True if the server settings are valid or False if otherwise. + """ - return (body, errors, warnings) + (server_name, + server_engine, + server_username, + server_password, + server_email, + server_update_frequency) = server_settings + + available_engines = _CLIENT_SERVERS.keys() + if "__init__" in available_engines: + available_engines.remove("__init__") + if server_engine not in available_engines: + return False + + if not re.match( + r"^([0-9]+[wdhms]{1}){1,5}$", + server_update_frequency + ): + return False + + return True diff --git a/modules/bibsword/lib/bibsword_client_dblayer.py b/modules/bibsword/lib/bibsword_client_dblayer.py index f52dc74fdf..4cf43e4a18 100644 --- a/modules/bibsword/lib/bibsword_client_dblayer.py +++ b/modules/bibsword/lib/bibsword_client_dblayer.py @@ -1,5 +1,7 @@ +# -*- coding: utf-8 -*- +# # This file is part of Invenio. -# Copyright (C) 2010, 2011 CERN. +# Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016 CERN. # # Invenio is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -15,349 +17,390 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. -''' -BibSWORD Client DBLayer -''' +""" +BibSword Client DBLayer. +""" -import datetime import time +from invenio.dateutils import convert_datestruct_to_datetext from invenio.dbquery import run_sql -from invenio.bibsword_config import CFG_SUBMISSION_STATUS_PUBLISHED, \ - CFG_SUBMISSION_STATUS_REMOVED -def get_remote_server_auth(id_remoteserver): - ''' - This function select the username and the password stored in the - table swrREMOTESERVER to execute HTTP Request - @param id_remoteserver: id of the remote server to contact - @return: (authentication_info) dictionnary conating username - password - ''' - - authentication_info = {'error':'', - 'hostname':'', - 'username':'', - 'password':'', - 'realm':'', - 'url_servicedocument':''} - - qstr = '''SELECT host, username, password, realm, url_servicedocument ''' \ - ''' FROM swrREMOTESERVER WHERE id=%s''' - qres = run_sql(qstr, (id_remoteserver, )) - - if len(qres) == 0 : - authentication_info['error'] = '''The server id doesn't correspond ''' \ - '''to any remote server''' - return authentication_info +# CREATE TABLE `swrCLIENTTEMPSUBMISSION` ( +# `id` varchar(128) NOT NULL, +# `object` longblob NOT NULL, +# `last_updated` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', +# PRIMARY KEY (`id`) +# ) + - (host, username, password, realm, url_servicedocument) = qres[0] +def store_temp_submission( + sid, + sobject_blob +): + """ + Store the temporary submission. + """ - authentication_info['hostname'] = host - authentication_info['username'] = username - authentication_info['password'] = password - authentication_info['realm'] = realm - authentication_info['url_servicedocument'] = url_servicedocument - - return authentication_info - - -def update_servicedocument(xml_servicedocument, id_remoteserver): - ''' - This function update the servicedocument filed containing all the - collections and categories for the given remote server - @param xml_servicedocument: xml file - @param id_remoteserver: id number of the remote server to update - @return: (boolean) true if update successfull, false else - ''' - - # get the current time to keep the last update time - current_type = datetime.datetime.now() - formatted_current_time = time.mktime(current_type.timetuple()) - - qstr = '''UPDATE swrREMOTESERVER ''' \ - '''SET xml_servicedocument=%s, last_update=%s ''' \ - '''WHERE id=%s''' - qres = run_sql(qstr, (xml_servicedocument, formatted_current_time, - id_remoteserver, )) - - return qres - - -def select_servicedocument(id_remoteserver): - ''' - This function retreive the servicedocument of the given remote server - @param id_remoteserver: id number of the remote server selected - @return: (xml_file) servicedocument xml file that contains coll and cat - ''' - - qstr = '''SELECT xml_servicedocument ''' \ - '''FROM swrREMOTESERVER ''' \ - '''WHERE id=%s''' - qres = run_sql(qstr, (id_remoteserver, )) - - if len(qres) == 0 : - return '' - - return qres[0][0] - - -def get_last_update(id_remoteserver): - ''' - This function return the last update time of the service document. This - is usefull to know if the service collection needs to be refreshed - @param id_remoteserver: id number of the remote server to check - @return: (datetime) datetime of the last update (yyyy-mm-dd hh:mm:ss) - ''' - - qstr = '''SELECT last_update ''' \ - '''FROM swrREMOTESERVER ''' \ - '''WHERE id=%s ''' - qres = run_sql(qstr, (id_remoteserver, )) - - if len(qres) == 0: - return '0' - - return qres[0][0] - - -def get_all_remote_server(id_server): - ''' - This function select the name of all remote service implementing the - SWORD protocol. It returns a list of dictionnary containing three fields: - id, name and host - @return: (remote_server) list of dictionnary (id - name - host) of each - remote server - ''' - - remote_servers = [] - - if id_server == '': - qstr = '''SELECT id, name, host FROM swrREMOTESERVER''' - qres = run_sql(qstr) - else : - qstr = ''' SELECT id, name, host FROM swrREMOTESERVER WHERE id=%s''' - qres = run_sql(qstr, (id_server, )) + query = """ + INSERT INTO swrCLIENTTEMPSUBMISSION + (id, object, last_updated) + VALUES (%s, %s, %s) + """ + now = convert_datestruct_to_datetext(time.localtime()) + params = (sid, sobject_blob, now) - for res in qres: - remote_server = {} - remote_server['id'] = res[0] - remote_server['name'] = res[1] - remote_server['host'] = res[2] - remote_servers.append(remote_server) + try: + res = run_sql(query, params) + except Exception: + return False + else: + return True - return remote_servers +def retrieve_temp_submission( + sid +): + """ + Retrieve the temporary submission. + """ -def is_record_sent_to_server(id_server, id_record): - ''' - check in the table swrCLIENTDATA that the current record has not already - been sent before - @param id_server: id of the remote server where to send the record - @param id_record: id of the record to send - return : True if a value was found, false else - ''' + query = """ + SELECT object + FROM swrCLIENTTEMPSUBMISSION + WHERE id=%s + """ - qstr = '''SELECT COUNT(*) FROM swrCLIENTDATA ''' \ - '''WHERE id_swrREMOTESERVER=%s AND id_record=%s ''' \ - '''AND status NOT LIKE 'removed' ''' - qres = run_sql(qstr, (id_server, id_record, )) - - if (qres[0][0] == 0): + params = (sid,) + + res = run_sql(query, params) + + if res: + return res[0][0] + else: + return None + + +def update_temp_submission( + sid, + sobject_blob +): + """ + Update the temporary submission. + """ + + query = """ + UPDATE swrCLIENTTEMPSUBMISSION + SET object=%s, + last_updated=%s + WHERE id=%s + """ + + now = convert_datestruct_to_datetext(time.localtime()) + params = (sobject_blob, now, sid) + + res = run_sql(query, params) + + return res + + +def delete_temp_submission( + sid +): + """ + Delete the temporary submission. + """ + + query = """ + DELETE FROM swrCLIENTTEMPSUBMISSION + WHERE id=%s + """ + + params = (sid,) + + res = run_sql(query, params) + + return res + + +# CREATE TABLE `swrCLIENTSUBMISSION` ( +# `id_user` int(15) unsigned NOT NULL, +# `id_record` mediumint(8) unsigned NOT NULL, +# `id_server` int(11) unsigned NOT NULL, +# `url_alternate` varchar(256) NOT NULL DEFAULT '', +# `url_edit` varchar(256) NOT NULL DEFAULT '', +# `status` varchar(256) NOT NULL DEFAULT '', +# `submitted` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', +# `last_updated` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', +# PRIMARY KEY (`id_record`,`id_server`) +# ) + + +def archive_submission( + user_id, + record_id, + server_id, + alternate_url, + edit_url +): + """ + Archive the given submission. + """ + + query = """ + INSERT INTO swrCLIENTSUBMISSION + ( + id_user, + id_record, + id_server, + url_alternate, + url_edit, + submitted + ) + VALUES ( + %s, + %s, + %s, + %s, + %s, + %s + ) + """ + + now = convert_datestruct_to_datetext(time.localtime()) + params = ( + user_id, + record_id, + server_id, + alternate_url, + edit_url, + now + ) + + try: + res = run_sql(query, params) + except Exception: return False else: return True -def insert_into_swr_clientdata(id_swr_remoteserver, - recid, - report_no, - remote_id, - id_user, - user_name, - user_email, - xml_media_deposit, - xml_metadata_submit, - link_media, - link_metadata, - link_status): - ''' - This method insert a new row in the swrCLIENTDATA table. Some are given in - parameters and some other such as the insertion time and the submission - status are set by default - @param id_swr_remoteserver: foreign key of the sword remote server - @param recid: foreign key of the submitted record - @param id_user: foreign key of the user who did the submission - @param xml_media_deposit: xml response after the media deposit - @param xml_metadata_submit: xml response after the metadata submission - @param remote_id: record id given by the remote server in the response - @param link_media: remote url where to find the depositted medias - @param link_metadata: remote url where to find the submitted metadata - @param link_status: remote url where to check the submission status - @return: (result) the id of the new row inserted, 0 if the submission - didn't work - ''' - - current_date = time.strftime("%Y-%m-%d %H:%M:%S") - xml_media_deposit.replace("\"", "\'") - xml_metadata_submit.encode('utf-8') - - qstr = '''INSERT INTO swrCLIENTDATA (id_swrREMOTESERVER, id_record, ''' \ - '''report_no, id_remote, id_user, user_name, user_email, ''' \ - '''xml_media_deposit, xml_metadata_submit, submission_date, ''' \ - '''link_medias, link_metadata, link_status, last_update) ''' \ - '''VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, ''' \ - '''%s, %s) ''' - qres = run_sql(qstr, (id_swr_remoteserver, recid, report_no, remote_id, - id_user, user_name, user_email, xml_media_deposit, - xml_metadata_submit, current_date, link_media, - link_metadata, link_status, current_date, )) - return qres - - -def count_nb_submitted_record() : - ''' - return : the amount of submitted records - ''' - - qstr = '''SELECT COUNT(*) FROM swrCLIENTDATA''' - qres = run_sql(qstr, ()) - - return qres[0][0] - - -def delete_from_swr_clientdata(id_submit): - ''' - delete the given row from the swrCLIENTDATA table. Used by the test suit - @param id_submit: id of the row to delete - result : boolean, true if deleted, false else - ''' - - qstr = ''' DELETE FROM swrCLIENTDATA WHERE id=%s ''' - qres = run_sql(qstr, (id_submit, )) - - return qres - - -def select_submitted_record_infos(first_row=0, offset=10, row_id=''): - ''' - this method return a bidimentionnal table containing all rows of the - table swrCLIENTDATA. If sepecified, the search can be limited to a - server, a record or a record on a server - @param first_row: give the limit where to start the selection - @param offset: give the maximal amount of rows to select - @return: table of row containing each colomn of the table swrCLIENTDATA - - FIXME: first_row is apparently supposed to select the chosen - id_swrREMOTESERVER, but it is currently strangely handled... - ''' - - wstr = '' - if row_id != '' : - wstr = '''WHERE d.id = %s ''' - qstr = '''SELECT d.id, d.id_swrREMOTESERVER, r.name, r.host , ''' \ - '''d.id_record, d.report_no, d.id_remote, d.id_user, ''' \ - '''d.user_name, d.user_email, d.submission_date, ''' \ - '''d.publication_date, d.removal_date, d.link_medias, ''' \ - '''d.link_metadata, d.link_status, d.status, r.url_base_record ''' \ - '''FROM swrCLIENTDATA as d inner join swrREMOTESERVER ''' \ - '''as r ON d.id_swrREMOTESERVER = r.id ''' + wstr + \ - '''ORDER BY d.last_update DESC LIMIT %s,%s''' - if wstr != '' : - qres = run_sql(qstr, (row_id, first_row, offset, )) - else : - qres = run_sql(qstr, (first_row, offset, )) - - results = [] - for res in qres : - result = {'publication_date':'', 'removal_date':''} - result['id'] = res[0] - result['id_server'] = res[1] - result['server_name'] = res[2] - result['server_host'] = res[3] - result['id_record'] = res[4] - result['report_no'] = res[5] - result['id_remote'] = res[6] - result['id_user'] = res[7] - result['user_name'] = res[8] - result['user_email'] = res[9] - result['submission_date'] = res[10].strftime("%Y-%m-%d %H:%M:%S") - if res[11] != None : - result['publication_date'] = res[11].strftime("%Y-%m-%d %H:%M:%S") - if res[12] != None : - result['removal_date'] = res[12].strftime("%Y-%m-%d %H:%M:%S") - result['link_medias'] = res[13] - result['link_metadata'] = res[14] - result['link_status'] = res[15] - result['status'] = res[16] - result['url_base_remote'] = res[17] - results.append(result) - - return results - - -def update_submission_status(id_record, status, remote_id=''): - ''' - update the submission field with the new status of the submission - @param id_record: id of the row to update - @param status: new value to set in the status field - @return: true if update done, else, false - ''' - - current_date = time.strftime("%Y-%m-%d %H:%M:%S") - - if status == CFG_SUBMISSION_STATUS_PUBLISHED and remote_id != '' : - qstr = '''UPDATE swrCLIENTDATA SET status=%s, id_remote=%s, ''' \ - '''publication_date=%s, last_update=%s WHERE id=%s ''' - qres = run_sql(qstr, (status, remote_id, current_date, current_date, - id_record, )) - - - if status == CFG_SUBMISSION_STATUS_REMOVED : - qstr = '''UPDATE swrCLIENTDATA SET status=%s, removal_date=%s, ''' \ - '''last_update=%s WHERE id=%s ''' - qres = run_sql(qstr, (status, current_date, current_date, id_record, )) - - else : - qstr = '''UPDATE swrCLIENTDATA SET status=%s, last_update=%s ''' \ - '''WHERE id=%s ''' - qres = run_sql(qstr, (status, current_date, id_record, )) - - return qres - - -def select_remote_server_infos(id_server): - ''' - Select fields of the given remote server and return it in a tuple - @param id_server: id of the server to select - @return: (server_info) tuple containing all the available infos - ''' - - server_info = {'server_id' : '', - 'server_name' : '', - 'server_host' : '', - 'username' : '', - 'password' : '', - 'email' : '', - 'realm' : '', - 'url_base_record' : '', - 'url_servicedocument' : ''} - - qstr = '''SELECT id, name, host, username, password, email, realm, ''' \ - '''url_base_record, url_servicedocument ''' \ - '''FROM swrREMOTESERVER WHERE id = %s ''' - qres = run_sql(qstr, (id_server, )) - - result = qres[0] - - server_info['server_id'] = result[0] - server_info['server_name'] = result[1] - server_info['server_host'] = result[2] - server_info['username'] = result[3] - server_info['password'] = result[4] - server_info['email'] = result[5] - server_info['realm'] = result[6] - server_info['url_base_record'] = result[7] - server_info['url_servicedocument'] = result[8] - - return server_info +def is_submission_archived( + record_id, + server_id, +): + """ + If the given record has already been archived to the given server + return True, otherwise return False. + """ + + query = """ + SELECT COUNT(id_record) + FROM swrCLIENTSUBMISSION + WHERE id_record=%s + AND id_server=%s + """ + + params = ( + record_id, + server_id, + ) + + res = run_sql(query, params) + + return bool(res[0][0]) + + +def get_submissions( + with_dict=False +): + """ + Get the Sword client submissions. + """ + + query = """ + SELECT user.nickname, + swrCLIENTSUBMISSION.id_record, + swrCLIENTSUBMISSION.id_server, + swrCLIENTSERVER.name, + swrCLIENTSUBMISSION.submitted, + swrCLIENTSUBMISSION.status, + swrCLIENTSUBMISSION.last_updated, + swrCLIENTSUBMISSION.url_alternate + FROM swrCLIENTSUBMISSION + JOIN swrCLIENTSERVER + ON swrCLIENTSUBMISSION.id_server = swrCLIENTSERVER.id + JOIN user + ON swrCLIENTSUBMISSION.id_user = user.id + ORDER BY swrCLIENTSUBMISSION.submitted DESC + """ + + params = None + + result = run_sql(query, params, with_dict=with_dict) + + return result + + +# CREATE TABLE `swrCLIENTSERVER` ( +# `id` int(11) unsigned NOT NULL AUTO_INCREMENT, +# `name` varchar(64) NOT NULL, +# `engine` varchar(64) NOT NULL, +# `username` varchar(64) NOT NULL, +# `password` varchar(64) NOT NULL, +# `email` varchar(64) NOT NULL, +# `service_document_parsed` longblob, +# `update_frequency` varchar(16) NOT NULL, +# `last_updated` timestamp DEFAULT 0, +# PRIMARY KEY (`id`) +# ) + + +def get_servers( + server_id=None, + with_dict=False +): + """ + Get the Sword client servers. + + Given a server_id, get that server only. + """ + + query = """ + SELECT id AS server_id, + name, + engine, + username, + password, + email, + service_document_parsed, + update_frequency, + last_updated + FROM swrCLIENTSERVER + """ + + if server_id is not None: + query += """ + WHERE id=%s + """ + params = (server_id,) + else: + params = None + + result = run_sql(query, params, with_dict=with_dict) + + return result + + +def delete_servers( + server_id=None +): + """ + Delete the Sword client servers. + + Given a server_id, delete that server only. + """ + + query = """ + DELETE + FROM swrCLIENTSERVER + """ + + if server_id is not None: + query += """ + WHERE id=%s + """ + params = (server_id,) + else: + params = None + + result = run_sql(query, params) + + return result + + +def modify_server( + server_id, + server_name, + server_engine, + server_username, + server_password, + server_email, + server_update_frequency +): + """ + Modify the Sword client server. + + Given a server_id. + """ + + query = """ + UPDATE swrCLIENTSERVER + SET name=%s, + engine=%s, + username=%s, + password=%s, + email=%s, + update_frequency=%s + WHERE id=%s + """ + + params = ( + server_name, + server_engine, + server_username, + server_password, + server_email, + server_update_frequency, + server_id, + ) + + result = run_sql(query, params) + + return result + + +def add_server( + server_name, + server_engine, + server_username, + server_password, + server_email, + server_update_frequency +): + """ + Add a Sword client server based on the given information. + """ + + query = """ + INSERT INTO swrCLIENTSERVER + (name, + engine, + username, + password, + email, + update_frequency) + VALUES (%s, + %s, + %s, + %s, + %s, + %s) + """ + + params = ( + server_name, + server_engine, + server_username, + server_password, + server_email, + server_update_frequency, + ) + + try: + result = run_sql(query, params) + except: + result = None + + return result diff --git a/modules/bibsword/lib/bibsword_client_formatter.py b/modules/bibsword/lib/bibsword_client_formatter.py deleted file mode 100644 index 1f10c94426..0000000000 --- a/modules/bibsword/lib/bibsword_client_formatter.py +++ /dev/null @@ -1,1196 +0,0 @@ -##This file is part of Invenio. -# Copyright (C) 2010, 2011 CERN. -# -# Invenio is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of the -# License, or (at your option) any later version. -# -# Invenio is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Invenio; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. - -''' -BibSWORD Client Formatter -''' - -import zipfile -import os -from tempfile import mkstemp -from xml.dom import minidom -from invenio.config import CFG_TMPDIR -from invenio.bibtask import task_low_level_submission -from invenio.bibsword_config import CFG_MARC_REPORT_NUMBER, \ - CFG_MARC_TITLE, \ - CFG_MARC_AUTHOR_NAME, \ - CFG_MARC_AUTHOR_AFFILIATION, \ - CFG_MARC_CONTRIBUTOR_NAME, \ - CFG_MARC_CONTRIBUTOR_AFFILIATION, \ - CFG_MARC_ABSTRACT, \ - CFG_MARC_ADDITIONAL_REPORT_NUMBER, \ - CFG_MARC_DOI, \ - CFG_MARC_JOURNAL_REF_CODE, \ - CFG_MARC_JOURNAL_REF_TITLE, \ - CFG_MARC_JOURNAL_REF_PAGE, \ - CFG_MARC_JOURNAL_REF_YEAR, \ - CFG_MARC_COMMENT, \ - CFG_MARC_RECORD_SUBMIT_INFO, \ - CFG_SUBMIT_ARXIV_INFO_MESSAGE, \ - CFG_DOCTYPE_UPLOAD_COLLECTION, \ - CFG_SUBMISSION_STATUS_SUBMITTED, \ - CFG_SUBMISSION_STATUS_PUBLISHED, \ - CFG_SUBMISSION_STATUS_ONHOLD, \ - CFG_SUBMISSION_STATUS_REMOVED -from invenio.bibdocfile import BibRecDocs -from invenio.bibformat_engine import BibFormatObject - -#------------------------------------------------------------------------------- -# Formating servicedocument file -#------------------------------------------------------------------------------- - -def format_remote_server_infos(servicedocument): - ''' - Get all informations about the server's options such as SWORD version, - maxUploadSize, ... These informations are found in the servicedocument - of the given server - @param servicedocument: xml servicedocument in a string format - @return: server_infomation. tuple containing the version, the - maxUploadSize and the available modes - ''' - - #contains information tuple {'version', 'maxUploadSize', 'verbose', 'noOp'} - server_informations = {'version' : '', - 'maxUploadSize' : '', - 'verbose' : '', - 'noOp' : '', - 'error' : '' } - - # now the xml node are accessible by programation - try: - parsed_xml_collections = minidom.parseString(servicedocument) - except IOError: - server_informations['error'] = \ - 'No servicedocument found for the remote server' - return server_informations - - # access to the root of the xml file - xml_services = parsed_xml_collections.getElementsByTagName('service') - xml_service = xml_services[0] - - # get value of the node - version_node = xml_service.getElementsByTagName('sword:version')[0] - server_informations['version'] = \ - version_node.firstChild.nodeValue.encode('utf-8') - - # get value of the node - max_upload_node = xml_service.getElementsByTagName('sword:maxUploadSize')[0] - server_informations['maxUploadSize'] = \ - max_upload_node.firstChild.nodeValue.encode('utf-8') - - - # get value of the node - verbose_node = xml_service.getElementsByTagName('sword:verbose')[0] - server_informations['verbose'] = \ - verbose_node.firstChild.nodeValue.encode('utf-8') - - # get value of the node - no_op_node = xml_service.getElementsByTagName('sword:noOp')[0] - server_informations['noOp'] = \ - no_op_node.firstChild.nodeValue.encode('utf-8') - - return server_informations - - -def format_remote_collection(servicedocument): - ''' - The function parse the servicedocument document and return a list with - the collections of the given file ['id', 'name', 'url'] - @param servicedocument: xml file returned by the remote server. - @return: the list of collection found in the service document - ''' - - collections = [] # contains list of collection tuple {'id', 'url', 'label'} - - # get the collections root node - collection_nodes = parse_xml_servicedocument_file(servicedocument) - - # i will be the id of the collection - i = 1 - - #--------------------------------------------------------------------------- - # recuperation of the collections - #--------------------------------------------------------------------------- - - # loop that goes in each node's collection of the document - for collection_node in collection_nodes: - - # dictionnary that contains the collections - collection = {} - - collection['id'] = str(i) - i = i + 1 - - # collection uri (where to deposit the media) - collection['url'] = \ - collection_node.attributes['href'].value.encode('utf-8') - - # collection name that is displayed to the user - xml_title = collection_node.getElementsByTagName('atom:title') - collection['label'] = xml_title[0].firstChild.nodeValue.encode('utf-8') - - # collection added to the collections list - collections.append(collection) - - return collections - - -def format_collection_informations(servicedocument, id_collection): - ''' - This methode parse the given servicedocument to find the given collection - node. Then it retrieve all information about the collection that contains - the collection node. - @param servicedocument: xml file returned by the remote server. - @param id_collection: position of the collection in the sd (1 = first) - @return: (collection_informations) tuple containing infos - ''' - - # contains information tuple {[accept], 'collectionPolicy', 'mediation', - # 'treatment', 'accept_packaging'} - collection_informations = {} - - # get the collections root node - collection_nodes = parse_xml_servicedocument_file(servicedocument) - - # recuperation of the selected collection - collection_node = collection_nodes[int(id_collection)-1] - - # get value of the nodes - accept_nodes = collection_node.getElementsByTagName('accept') - accept = [] - for accept_node in accept_nodes: - accept.append(accept_node.firstChild.nodeValue.encode('utf-8')) - - collection_informations['accept'] = accept - - # get value of the nodes - collection_policy = \ - collection_node.getElementsByTagName('sword:collectionPolicy')[0] - collection_informations['collectionPolicy'] = \ - collection_policy.firstChild.nodeValue.encode('utf-8') - - # get value of the nodes - mediation = collection_node.getElementsByTagName('sword:mediation')[0] - collection_informations['mediation'] = \ - mediation.firstChild.nodeValue.encode('utf-8') - - # get value of the nodes - treatment = collection_node.getElementsByTagName('sword:treatment')[0] - collection_informations['treatment'] = \ - treatment.firstChild.nodeValue.encode('utf-8') - - # get value of the nodes - accept_packaging = \ - collection_node.getElementsByTagName('sword:acceptPackaging')[0] - collection_informations['accept_packaging'] = \ - accept_packaging.firstChild.nodeValue.encode('utf-8') - - return collection_informations - - -def format_primary_categories(servicedocument, collection_id=0): - ''' - This method parse the servicedocument to retrieve the primary category - of the given collection. If no collection is given, it takes the first - one. - @param servicedocument: xml file returned by the remote server. - @param collection_id: id of the collection to search - @return: list of primary categories tuple ('id', 'url', 'label') - ''' - - categories = [] # contains list of category tuple {'id', 'url', 'label'} - - # get the collections root node - collection_nodes = parse_xml_servicedocument_file(servicedocument) - - # i will be the id of the collection - i = 1 - - # recuperation of the selected collection - collection_node = collection_nodes[int(collection_id)-1] - - #--------------------------------------------------------------------------- - # recuperation of the categories - #--------------------------------------------------------------------------- - - # select all primary category nodes - primary_categories_node = \ - collection_node.getElementsByTagName('arxiv:primary_categories')[0] - primary_category_nodes = \ - primary_categories_node.getElementsByTagName('arxiv:primary_category') - - # loop that goes in each primary_category nodes - for primary_category_node in primary_category_nodes: - - # dictionnary that contains the categories - category = {} - - category['id'] = str(i) - i = i + 1 - - category['url'] = \ - primary_category_node.attributes['term'].value.encode('utf-8') - category['label'] = \ - primary_category_node.attributes['label'].value.encode('utf-8') - - categories.append(category) - - return categories - - -def format_secondary_categories(servicedocument, collection_id=0): - ''' - This method parse the servicedocument to retrieve the optional categories - of the given collection. If no collection is given, it takes the first - one. - @param servicedocument: xml file returned by the remote server. - @param collection_id: id of the collection to search - @return: list of optional categories tuple ('id', 'url', 'label') - ''' - - categories = [] # contains list of category tuple {'id', 'url', 'label'} - - # get the collections root node - collection_nodes = parse_xml_servicedocument_file(servicedocument) - - # i will be the id of the collection - i = 1 - - # recuperation of the selected collection - collection_id = int(collection_id) - 1 - collection_node = collection_nodes[int(collection_id)] - - #--------------------------------------------------------------------------- - # recuperation of the categories - #--------------------------------------------------------------------------- - - # select all primary category nodes - categories_node = collection_node.getElementsByTagName('categories')[0] - category_nodes = categories_node.getElementsByTagName('category') - - # loop that goes in each primary_category nodes - for category_node in category_nodes: - - # dictionnary that contains the categories - category = {} - - category['id'] = str(i) - i = i + 1 - - category['url'] = category_node.attributes['term'].value.encode('utf-8') - category['label'] = \ - category_node.attributes['label'].value.encode('utf-8') - - categories.append(category) - - return categories - - -def parse_xml_servicedocument_file(servicedocument): - ''' - This method parse a string containing a servicedocument to retrieve the - collection node. It is used by all function that needs to work with - collections - @param servicedocument: xml file in containing in a string - @return: (collecion_node) root node of all collecions - ''' - - # now the xml node are accessible by programation - parsed_xml_collections = minidom.parseString(servicedocument) - - # access to the root of the xml file - xml_services = parsed_xml_collections.getElementsByTagName('service') - xml_service = xml_services[0] - - # their is only the global workspace in this xml document - xml_workspaces = xml_service.getElementsByTagName('workspace') - xml_workspace = xml_workspaces[0] - - # contains all collections in the xml file - collection_nodes = xml_workspace.getElementsByTagName('collection') - - return collection_nodes - - -#------------------------------------------------------------------------------- -# Formating marcxml file -#------------------------------------------------------------------------------- - -def get_report_number_from_macrxml(marcxml): - ''' - retrieve the record id stored in the marcxml file. The record is in the - tag 'RECORD ID' - @param marcxml: marcxml file where to look for the record id - @return: the record id in a string - ''' - - #get the reportnumber tag list - tag = CFG_MARC_REPORT_NUMBER - if tag == '': - return '' - - #variable that contains the result of the parsing of the marcxml file - datafields = get_list_of_marcxml_datafields(marcxml) - - for datafield in datafields: - - report_number = get_subfield_value_from_datafield(datafield, tag) - if report_number != '': - return report_number - - return '' - - -def get_medias_to_submit(media_paths): - ''' - This method get a list of recod of submission. It format a list of - media containing name, size, type and file for each media id - @param media_paths: list of path to the media to upload - @return: list of media tuple - ''' - - # define the return value - media = {} - - fp = open("/tmp/test.txt", "w") - fp.write(media_paths[0]) - - if len(media_paths) > 1: - media_paths = format_file_to_zip_archiv(media_paths) - else: - media_paths = media_paths[0] - - if media_paths != '': - media['file'] = open(media_paths, "r").read() - media['size'] = len(media['file']) - media['name'] = media_paths.split('/')[-1].split(';')[0] - media['type'] = 'application/%s' % media['name'].split('.')[-1] - - return media - - -def get_media_from_recid(recid): - ''' - This method get the file in the given url - @param recid: id of the file to get - ''' - - medias = [] - - bibarchiv = BibRecDocs(recid) - bibdocs = bibarchiv.list_latest_files() - - for bibdocfile in bibdocs: - - bibfile = {'name': bibdocfile.get_full_name(), - 'file': '', - 'type': 'application/%s' % \ - bibdocfile.get_superformat().split(".")[-1], - 'path': bibdocfile.get_full_path(), - 'collection': bibdocfile.get_type(), - 'size': bibdocfile.get_size(), - 'loaded': False, - 'selected': ''} - - if bibfile['collection'] == "Main": - bibfile['selected'] = 'checked=yes' - - medias.append(bibfile) - - return medias - - -def format_author_from_marcxml(marcxml): - ''' - This method parse the marcxml file to retrieve the author of a document - @param marcxml: the xml file to parse - @return: tuple containing {'name', 'email' and 'affiliations'} - ''' - - #get the tag id for the given field - main_author = CFG_MARC_AUTHOR_NAME - main_author_affiliation = CFG_MARC_AUTHOR_AFFILIATION - - #variable that contains the result of the parsing of the marcxml file - datafields = get_list_of_marcxml_datafields(marcxml) - - #init the author tuple - author = {'name':'', 'email':'', 'affiliation':[]} - - for datafield in datafields: - - # retreive the main author - if author['name'] == '': - name = get_subfield_value_from_datafield(datafield, main_author) - if name != '': - author['name'] = name - - affiliation = get_subfield_value_from_datafield(datafield, main_author_affiliation) - if affiliation != '': - author['affiliation'].append(affiliation) - - return author - - -def format_marcxml_file(marcxml, is_file=False): - ''' - Parse the given marcxml file to retreive the metadata needed by the - forward of the document to ArXiv.org - @param marcxml: marxml file that contains metadata from Invenio - @return: (dictionnary) couple of key value needed for the push - ''' - - #init the return tuple - marcxml_values = { 'id' : '', - 'title' : '', - 'summary' : '', - 'contributors' : [], - 'journal_refs' : [], - 'report_nos' : [], - 'comment' : '', - 'doi' : '' } - - # check if the marcxml is not empty - if marcxml == '': - marcxml_values['error'] = "MARCXML string is empty !" - return marcxml_values - - #get the tag id and code from tag table - main_report_number = CFG_MARC_REPORT_NUMBER - add_report_number = CFG_MARC_ADDITIONAL_REPORT_NUMBER - main_title = CFG_MARC_TITLE - main_summary = CFG_MARC_ABSTRACT - main_author = CFG_MARC_AUTHOR_NAME - main_author_affiliation = CFG_MARC_AUTHOR_AFFILIATION - add_author = CFG_MARC_CONTRIBUTOR_NAME - add_author_affiliation = CFG_MARC_CONTRIBUTOR_AFFILIATION - main_comment = CFG_MARC_COMMENT - doi = CFG_MARC_DOI - journal_ref_code = CFG_MARC_JOURNAL_REF_CODE - journal_ref_title = CFG_MARC_JOURNAL_REF_TITLE - journal_ref_page = CFG_MARC_JOURNAL_REF_PAGE - journal_ref_year = CFG_MARC_JOURNAL_REF_YEAR - - #init tmp values - contributor = {'name' : '', 'email' : '', 'affiliation' : []} - - try: - bfo = BibFormatObject(recID=None, xml_record=marcxml) - except: - marcxml_values['error'] = "Unable to open marcxml file !" - return marcxml_values - - marcxml_values = { 'id' : bfo.field(main_report_number), - 'title' : bfo.field(main_title), - 'summary' : bfo.field(main_summary), - 'report_nos' : bfo.fields(add_report_number), - 'contributors' : [], - 'journal_refs' : [], - 'comment' : bfo.field(main_comment), - 'doi' : bfo.field(doi)} - - authors = bfo.fields(main_author[:-1], repeatable_subfields_p=True) - for author in authors: - name = author.get(main_author[-1], [''])[0] - affiliation = author.get(main_author_affiliation[-1], []) - author = {'name': name, 'email': '', 'affiliation': affiliation} - marcxml_values['contributors'].append(author) - - authors = bfo.fields(add_author[:-1], repeatable_subfields_p=True) - for author in authors: - name = author.get(add_author[-1], [''])[0] - affiliation = author.get(add_author_affiliation[-1], []) - author = {'name': name, 'email': '', 'affiliation': affiliation} - marcxml_values['contributors'].append(author) - - journals = bfo.fields(journal_ref_title[:-1]) - for journal in journals: - journal_title = journal.get(journal_ref_title[-1], '') - journal_page = journal.get(journal_ref_page[-1], '') - journal_code = journal.get(journal_ref_code[-1], '') - journal_year = journal.get(journal_ref_year[-1], '') - journal = "%s: %s (%s) pp. %s" % (journal_title, journal_code, journal_year, journal_page) - marcxml_values['journal_refs'].append(journal) - - return marcxml_values - - -def get_subfield_value_from_datafield(datafield, field_tag): - ''' - This function get the datafield note from a marcxml and get the tag - value according to the tag id and code given - @param datafield: xml node to be parsed - @param field_tag: tuple containing id and code to find - @return: value of the tag as a string - ''' - - # extract the tag number - tag = datafield.attributes["tag"] - - tag_id = field_tag[0] + field_tag[1] + field_tag[2] - tag_code = field_tag[5] - - # retreive the reference to the media - if tag.value == tag_id: - subfields = datafield.getElementsByTagName('subfield') - for subfield in subfields: - if subfield.attributes['code'].value == tag_code: - return subfield.firstChild.nodeValue.encode('utf-8') - - return '' - - -def get_list_of_marcxml_datafields(marcxml, isfile=False): - ''' - This method parse the marcxml file to retrieve the root of the datafields - needed by all function that format marcxml nodes. - @param marcxml: file or string that contains the marcxml file - @param isfile: boolean that informs if a file or a string was given - @return: root of all datafileds - ''' - - #variable that contains the result of the parsing of the marcxml file - if isfile: - try: - parsed_marcxml = minidom.parse(marcxml) - except IOError: - return 0 - else: - parsed_marcxml = minidom.parseString(marcxml) - - collections = parsed_marcxml.getElementsByTagName('collection') - - # some macxml file has no collection root but direct record entry - if len(collections) > 0: - collection = collections[0] - records = collection.getElementsByTagName('record') - else: - records = parsed_marcxml.getElementsByTagName('record') - - record = records[0] - - return record.getElementsByTagName('datafield') - - -def format_file_to_zip_archiv(paths): - ''' - This method takes a list of different type of file, zip its and group - its into a zip archiv for sending - @param paths: list of path to file of different types - @return: (zip archiv) zipped file that contains all fulltext to submit - ''' - - (zip_fd, zip_path) = mkstemp(suffix='.zip', prefix='bibsword_media_', - dir=CFG_TMPDIR) - - archiv = zipfile.ZipFile(zip_path, "w") - - for path in paths: - if os.path.exists(path): - archiv.write(path, os.path.basename(path), zipfile.ZIP_DEFLATED) - - archiv.close() - - return zip_path - - -#------------------------------------------------------------------------------- -# getting info from media deposit response file -#------------------------------------------------------------------------------- - -def format_link_from_result(result): - ''' - This method parses the xml file returned after the submission of a media - and retreive the URL contained in it - @param result: xml file returned by ArXiv - @return: (links) table of url - ''' - if isinstance(result, list): - result = result[0] - - # parse the xml to access each node - parsed_result = minidom.parseString(result) - - # finding the links in the xml file - xml_entries = parsed_result.getElementsByTagName('entry') - xml_entry = xml_entries[0] - xml_contents = xml_entry.getElementsByTagName('content') - - # getting the unique content node - content = xml_contents[0] - - # declare the dictionnary that contains type and url of a link - link = {} - link['link'] = content.attributes['src'].value.encode('utf-8') - link['type'] = content.attributes['type'].value.encode('utf-8') - - return link - - -def format_update_time_from_result(result): - ''' - parse any xml response to retreive and format the value of the 'updated' - tag. - @param result: xml result of a deposit or a submit call to a server - @return: formated date content in the node - ''' - - # parse the xml to access each node - parsed_result = minidom.parseString(result) - - # finding the links in the xml file - xml_entries = parsed_result.getElementsByTagName('entry') - xml_entry = xml_entries[0] - xml_updated = xml_entry.getElementsByTagName('updated') - - # getting the unique content node - updated = xml_updated[0] - - return updated.firstChild.nodeValue.encode('utf-8') - - -def format_links_from_submission(submission): - ''' - parse the xml response of a metadata submission and retrieve all the - informations proper to the link toward the media, the metadata and - the status - @param submission: xml response of a submission - @return: tuple { 'medias', 'metadata', 'status' } - ''' - - # parse the xml to access each node - parsed_result = minidom.parseString(submission) - - # finding the links in the xml file - xml_entries = parsed_result.getElementsByTagName('entry') - xml_entry = xml_entries[0] - xml_links = xml_entry.getElementsByTagName('link') - - # getting all content nodes - links = {'media':'', 'metadata':'', 'status':''} - - for link in xml_links: - - # declare the dictionnary that contains type and url of a link - if link.attributes['rel'].value == 'edit-media': - if links['media'] == '': - links['media'] = link.attributes['href'].value.encode('utf-8') - else: - links['media'] = links['media'] + ', ' + \ - link.attributes['href'].value.encode('utf-8') - - if link.attributes['rel'].value == 'edit': - links['metadata'] = link.attributes['href'].value.encode('utf-8') - - if link.attributes['rel'].value == 'alternate': - links['status'] = link.attributes['href'].value.encode('utf-8') - - return links - - -def format_id_from_submission(submission): - ''' - Parse the submission file to retrieve the arxiv id retourned - @param submission: xml file returned after the submission - @return: string containing the arxiv id - ''' - - # parse the xml to access each node - parsed_result = minidom.parseString(submission) - - # finding the id in the xml file - xml_entries = parsed_result.getElementsByTagName('entry') - xml_entry = xml_entries[0] - xml_id = xml_entry.getElementsByTagName('id')[0] - - remote_id = xml_id.firstChild.nodeValue.encode('utf-8') - - (begin, sep, end) = remote_id.rpartition("/") - - remote_id = 'arXiv:' - i = 0 - for elt in end: - remote_id += elt - if i == 3: - remote_id += '.' - i = i + 1 - - return remote_id - - -#------------------------------------------------------------------------------- -# write information in the marc file -#------------------------------------------------------------------------------- - -def update_marcxml_with_remote_id(recid, remote_id, action="append"): - ''' - Write a new entry in the given marc file. This entry is the remote record - id given by the server where the submission has been done - @param remote_id: the string containing the id to add to the marc file - return: boolean true if update done, false if problems - ''' - - field_tag = CFG_MARC_ADDITIONAL_REPORT_NUMBER - tag_id = "%s%s%s" % (field_tag[0], field_tag[1], field_tag[2]) - tag_code = field_tag[5] - - # concatenation of the string to append to the marc file - node = ''' - %(recid)s - - %(remote_id)s - -''' % { - 'recid': recid, - 'tagid': tag_id, - 'tagcode': tag_code, - 'remote_id': remote_id - } - - # creation of the tmp file containing the xml node to append - (tmpfd, filename) = mkstemp(suffix='.xml', prefix='bibsword_append_remote_id_', - dir=CFG_TMPDIR) - tmpfile = os.fdopen(tmpfd, 'w') - tmpfile.write(node) - tmpfile.close() - - # insert a task in bibsched to add the node in the marc file - if action == 'append': - result = \ - task_low_level_submission('bibupload', 'BibSword', '-a', filename) - elif action == 'delete': - result = \ - task_low_level_submission('bibupload', 'BibSword', '-d', filename) - - return result - - -def update_marcxml_with_info(recid, username, current_date, remote_id, - action='append'): - ''' - This function add a field in the marc file to informat that the - record has been submitted to a remote server - @param recid: id of the record to update - ''' - - # concatenation of the string to append to the marc file - node = ''' - %(recid)s - - %(submit_info)s - -''' % { - 'recid': recid, - 'tag': CFG_MARC_RECORD_SUBMIT_INFO, - 'submit_info': CFG_SUBMIT_ARXIV_INFO_MESSAGE % (username, current_date, remote_id) - } - - # creation of the tmp file containing the xml node to append - (tmpfd, filename) = mkstemp(suffix='.xml', prefix='bibsword_append_submit_info_', - dir=CFG_TMPDIR) - tmpfile = os.fdopen(tmpfd, 'w') - tmpfile.write(node) - tmpfile.close() - - # insert a task in bibschedul to add the node in the marc file - if action == 'append': - result = \ - task_low_level_submission('bibupload', 'BibSword', '-a', filename) - elif action == 'delete': - result = \ - task_low_level_submission('bibupload', 'BibSword', '-d', filename) - - return result - - - -def upload_fulltext(recid, path): - ''' - This method save the uploaded file to associated record - @param recid: id of the record - @param path: uploaded document to store - ''' - - # upload the file to the record - - bibarchiv = BibRecDocs(recid) - docname = path.split('/')[-1].split('.')[0] - doctype = path.split('.')[-1].split(';')[0] - bibarchiv.add_new_file(path, CFG_DOCTYPE_UPLOAD_COLLECTION, docname, - format=doctype) - - return '' - - -#------------------------------------------------------------------------------- -# work with the remote submission status xml file -#------------------------------------------------------------------------------- - -def format_submission_status(status_xml): - ''' - This method parse the given atom xml status string and retrieve the - the value of the tag - @param status_xml: xml atom entry - @return: dictionnary containing status, id and/or possible error - ''' - - result = {'status':'', 'id_submission':'', 'error':''} - - parsed_status = minidom.parseString(status_xml) - deposit = parsed_status.getElementsByTagName('deposit')[0] - status_node = deposit.getElementsByTagName('status')[0] - if status_node.firstChild != None: - status = status_node.firstChild.nodeValue.encode('utf-8') - else: - result['status'] = '' - return result - - #status = "submitted" - if status == CFG_SUBMISSION_STATUS_SUBMITTED: - result['status'] = status - return result - - #status = "published" - if status == CFG_SUBMISSION_STATUS_PUBLISHED: - result['status'] = status - arxiv_id_node = deposit.getElementsByTagName('arxiv_id')[0] - result['id_submission'] = \ - arxiv_id_node.firstChild.nodeValue.encode('utf-8') - return result - - #status = "onhold" - if status == CFG_SUBMISSION_STATUS_ONHOLD: - result['status'] = status - return result - - #status = "removed" - if status == 'unknown': - result['status'] = CFG_SUBMISSION_STATUS_REMOVED - error_node = deposit.getElementsByTagName('error')[0] - result['error'] = error_node.firstChild.nodeValue.encode('utf-8') - return result - - return result - - -#------------------------------------------------------------------------------- -# Classes for the generation of XML Atom entry containing submission metadata -#------------------------------------------------------------------------------- - -class BibSwordFormat: - ''' - This class gives the methodes needed to format all mandatories xml atom - entry nodes. It is extended by subclasses that has optional nodes add - to the standard SWORD format - ''' - - def __init__(self): - ''' No init necessary for this class ''' - - def frmt_id(self, recid): - ''' - This methode check if there is an id for the resource. If it is the case, - it format it returns a formated id node that may be inserted in the - xml metadata file - @param recid: the id of the resource - @return: (xml) xml node correctly formated - ''' - - if recid != '': - return '''%s\n''' % recid - return '' - - - def frmt_title(self, title): - ''' - This methode check if there is a title for the resource. If yes, - it returns a formated title node that may be inserted in the - xml metadata file - @param title: the title of the resource - @return: (xml) xml node correctly formated - ''' - - if title != '': - return '''%s\n''' % title - return '' - - - def frmt_author(self, author_name, author_email): - ''' - This methode check if there is a submitter for the resource. If yes, - it returns a formated author node that may containing the name and - the email of the author to be inserted in the xml metadata file - @param author_name: the name of the submitter of the resource - @param author_email: the email where the remote server send answers - @return: (xml) xml node correctly formated - ''' - - author = '' - if author_name != '': - author += '''\n''' - author += '''%s\n''' % author_name - if author_email != '': - author += '''%s\n''' % author_email - author += '''\n''' - return author - - - def frmt_summary(self, summary): - ''' - This methode check if there is a summary for the resource. If yes, - it returns a formated summary node that may be inserted in the - xml metadata file - @param summary: the summary of the resource - @return: (xml) xml node correctly formated - ''' - - if summary != '': - return '''%s\n''' % summary - return '' - - - def frmt_categories(self, categories, scheme): - ''' - This method check if there is some categories for the resource. If it - is the case, it returns the categorie nodes formated to be insered in - the xml metadata file - @param categories: list of categories for one resource - @return: (xml) xml node(s) correctly formated - ''' - - output = '' - - for category in categories: - - output += '''\n''' % (category['url'], scheme, category['label']) - - return output - - - def frmt_link(self, links): - ''' - This method check if there is some links for the resource. If it - is the case, it returns the links nodes formated to be insered in - the xml metadata file - @param links: list of links for the resource - @return: (xml) xml node(s) correctly formated - ''' - - output = '' - - if links != '': - output += '''\n''' % links['type'] - - return output - - - -class ArXivFormat(BibSwordFormat): - ''' - This class inherit from the class BibSwordFormat. It add some specific - mandatory nodes to the standard SWORD format. - ''' - - #--------------------------------------------------------------------------- - # Formating metadata file for submission - #--------------------------------------------------------------------------- - - def format_metadata(self, metadata): - ''' - This method format an atom file that fits with the arxiv atom format - used for the subission of the metadata during the push to arxiv process. - @param metadata: tuple containing every needed information + some optional - @return: (xml file) arxiv atom file - ''' - - #----------------------------------------------------------------------- - # structure of the arxiv metadata submission atom entry - #----------------------------------------------------------------------- - - output = '''\n''' - output += '''\n''' - - #id - if 'id' in metadata: - output += BibSwordFormat.frmt_id(self, metadata['id']) - - #title - if 'title' in metadata: - output += BibSwordFormat.frmt_title(self, - metadata['title']) - - #author - if 'author_name' in metadata and 'author_email' in metadata: - output += BibSwordFormat.frmt_author(self, metadata['author_name'], - metadata['author_email']) - - #contributors - if 'contributors' in metadata: - output += '' + self.frmt_contributors(metadata['contributors']) - - #summary - if 'summary' in metadata: - output += BibSwordFormat.frmt_summary(self, metadata['summary']) - - #categories - if 'categories' in metadata: - output += BibSwordFormat.frmt_categories(self, metadata['categories'], - 'http://arxiv.org/terms/arXiv/') - - #primary_category - if 'primary_url' in metadata and 'primary_label' in metadata: - output += self.frmt_primary_category(metadata['primary_url'], - metadata['primary_label'], - 'http://arxiv.org/terms/arXiv/') - - #comment - if 'comment' in metadata: - output += self.frmt_comment(metadata['comment']) - - #journal references - if 'journal_refs' in metadata: - output += self.frmt_journal_ref(metadata['journal_refs']) - - #report numbers - if 'report_nos' in metadata: - output += self.frmt_report_no(metadata['report_nos']) - - #doi - if 'doi' in metadata: - output += self.frmt_doi(metadata['doi']) - - #link - if 'links' in metadata: - output += BibSwordFormat.frmt_link(self, metadata['links']) - - output += '''''' - - return output - - - def frmt_contributors(self, contributors): - ''' - This method display each contributors in the format of an editable input - text. This allows the user to modifie it. - @param contributors: The list of all contributors of the document - @return: (html code) the html code that display each dropdown list - ''' - - output = '' - - for contributor in contributors: - output += '''\n''' - output += '''%s\n''' % contributor['name'] - if contributor['email'] != '': - output += '''%s\n''' % \ - contributor['email'] - if len(contributor['affiliation']) != 0: - for affiliation in contributor['affiliation']: - output += '''%s'''\ - '''\n''' % affiliation - output += '''\n''' - - return output - - - def frmt_primary_category(self, primary_url, primary_label, scheme): - ''' - This method format the primary category as an element of a dropdown - list. - @param primary_url: url of the primary category deposit - @param primary_label: name of the primary category to display - @param scheme: url of the primary category schema - @return: html code containing each element to display - ''' - - output = '' - - if primary_url != '': - output += '''\n''' % (scheme, primary_label, primary_url) - - return output - - - def frmt_comment(self, comment): - ''' - This methode check if there is an comment given. If it is the case, it - format it returns a formated comment node that may be inserted in the xml - metadata file - @param comment: the string comment - @return: (xml) xml node correctly formated - ''' - - output = '' - - if comment != '': - output = '''%s\n''' % comment - - return output - - - def frmt_journal_ref(self, journal_refs): - ''' - This method check if there is some journal refs for the resource. If it - is the case, it returns the journal_ref nodes formated to be insered in - the xml metadata file - @param journal_refs: list of journal_refs for one resource - @return: (xml) xml node(s) correctly formated - ''' - - output = '' - - for journal_ref in journal_refs: - output += '''%s\n''' % \ - journal_ref - - return output - - - def frmt_report_no(self, report_nos): - ''' - This method check if there is some report numbres for the resource. If it - is the case, it returns the report_nos nodes formated to be insered in - the xml metadata file - @param report_nos: list of report_nos for one resource - @return: (xml) xml node(s) correctly formated - ''' - - output = '' - - for report_no in report_nos: - output += '''%s\n''' % \ - report_no - - return output - - - def frmt_doi(self, doi): - '''This methode check if there is an doi given. If it is the case, it - format it returns a formated doi node that may be inserted in the xml - metadata file - @param doi: the string doi - @return: (xml) xml node correctly formated - ''' - - output = '' - - if doi != '': - output = '''%s\n''' % doi - - return output diff --git a/modules/bibsword/lib/bibsword_client_http.py b/modules/bibsword/lib/bibsword_client_http.py deleted file mode 100644 index 96563c0ec0..0000000000 --- a/modules/bibsword/lib/bibsword_client_http.py +++ /dev/null @@ -1,188 +0,0 @@ -# This file is part of Invenio. -# Copyright (C) 2010, 2011 CERN. -# -# Invenio is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of the -# License, or (at your option) any later version. -# -# Invenio is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Invenio; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. - -''' -BibSWORD Client Http Queries -''' - -import urllib2 -from tempfile import NamedTemporaryFile -from invenio.config import CFG_TMPDIR -from invenio.urlutils import make_user_agent_string - -class RemoteSwordServer: - '''This class gives every tools to communicate with the SWORD/APP deposit - of ArXiv. - ''' - - # static variable used to properly perform http request - agent = make_user_agent_string("BibSWORD") - - - def __init__(self, authentication_infos): - - ''' - This method the constructor of the class, it initialise the - connection using a passord. That allows users to connect with - auto-authentication. - @param self: reference to the current instance of the class - @param authentication_infos: dictionary with authentication infos containing - keys: - - realm: realm of the server - - hostname: hostname of the server - - username: name of an arxiv known user - - password: password of the known user - ''' - - #password manager with default realm to avoid looking for it - passman = urllib2.HTTPPasswordMgrWithDefaultRealm() - - passman.add_password(authentication_infos['realm'], - authentication_infos['hostname'], - authentication_infos['username'], - authentication_infos['password']) - - #create an authentificaiton handler - authhandler = urllib2.HTTPBasicAuthHandler(passman) - - http_handler = urllib2.HTTPHandler(debuglevel=0) - - opener = urllib2.build_opener(authhandler, http_handler) - # insalling : every call to opener will user the same user/pass - urllib2.install_opener(opener) - - - def get_remote_collection(self, url): - ''' - This method sent a request to the servicedocument to know the - collections offer by arxives. - @param self: reference to the current instance of the class - @param url: the url where the request is made - @return: (xml file) collection of arxiv allowed for the user - ''' - - #format the request - request = urllib2.Request(url) - - #launch request - #try: - response = urllib2.urlopen(request) - #except urllib2.HTTPError: - # return '' - #except urllib2.URLError: - # return '' - - return response.read() - - - def deposit_media(self, media, collection, onbehalf): - ''' - This method allow the deposit of any type of media on a given arxiv - collection. - @param self: reference to the current instanc off the class - @param media: dict of file info {'type', 'size', 'file'} - @param collection: abreviation of the collection where to deposit - @param onbehalf: user that make the deposition - @return: (xml file) contains error ot the url of the temp file - ''' - - #format the final deposit URL - deposit_url = collection - - #prepare the header - headers = {} - headers['Content-Type'] = media['type'] - headers['Content-Length'] = media['size'] - #if on behalf, add to the header - if onbehalf != '': - headers['X-On-Behalf-Of'] = onbehalf - - headers['X-No-Op'] = 'True' - headers['X-Verbose'] = 'True' - headers['User-Agent'] = self.agent - - #format the request - result = urllib2.Request(deposit_url, media['file'], headers) - - #launch request - try: - return urllib2.urlopen(result).read() - except urllib2.HTTPError: - return '' - - - def metadata_submission(self, deposit_url, metadata, onbehalf): - ''' - This method send the metadata to ArXiv, then return the answere - @param metadata: xml file to submit to ArXiv - @param onbehalf: specify the persone (and email) to informe of the - publication - ''' - - #prepare the header of the request - headers = {} - headers['Host'] = 'arxiv.org' - headers['User-Agent'] = self.agent - headers['Content-Type'] = 'application/atom+xml;type=entry' - #if on behalf, add to the header - if onbehalf != '': - headers['X-On-Behalf-Of'] = onbehalf - - headers['X-No-Op'] = 'True' - headers['X-verbose'] = 'True' - - #format the request - result = urllib2.Request(deposit_url, metadata, headers) - - #launch request - try: - response = urllib2.urlopen(result).read() - except urllib2.HTTPError, e: - tmpfd = NamedTemporaryFile(mode='w', suffix='.xml', prefix='bibsword_error_', - dir=CFG_TMPDIR, delete=False) - tmpfd.write(e.read()) - tmpfd.close() - return '' - except urllib2.URLError: - return '' - - return response - - - def get_submission_status(self, status_url) : - ''' - This method get the xml file from the given URL and return it - @param status_url: url where to get the status - @return: xml atom entry containing the status - ''' - - #format the http request - request = urllib2.Request(status_url) - request.add_header('Host', 'arxiv.org') - request.add_header('User-Agent', self.agent) - - #launch request - try: - response = urllib2.urlopen(request).read() - except urllib2.HTTPError: - return 'HTTPError (Might be an authentication issue)' - except urllib2.URLError: - return 'Wrong url' - - return response - - diff --git a/modules/bibsword/lib/bibsword_client_server.py b/modules/bibsword/lib/bibsword_client_server.py new file mode 100644 index 0000000000..faef8ff092 --- /dev/null +++ b/modules/bibsword/lib/bibsword_client_server.py @@ -0,0 +1,521 @@ +# -*- coding: utf-8 -*- +# +# This file is part of Invenio. +# Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016 CERN. +# +# Invenio is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# Invenio is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Invenio; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +""" +Base class for the SWORD client servers. +""" + +import urllib2 +from socket import getdefaulttimeout, setdefaulttimeout +from mimetypes import guess_all_extensions +from tempfile import NamedTemporaryFile +from datetime import datetime, timedelta +import re + +from invenio.config import CFG_TMPDIR +from invenio.dbquery import( + run_sql, + serialize_via_marshal, + deserialize_via_marshal +) +from invenio.dateutils import( + convert_datestruct_to_datetext +) + +from invenio.bibsword_config import \ + CFG_BIBSWORD_USER_AGENT, \ + CFG_BIBSWORD_FILE_TYPES_MAPPING, \ + CFG_BIBSWORD_DEFAULT_TIMEOUT, \ + CFG_BIBSWORD_FORCE_DEFAULT_TIMEOUT, \ + CFG_BIBSWORD_ATOM_ENTRY_MIME_TYPE + + +class SwordClientServer(object): + """ + Base class for the SWORD client servers. + Missing funcionality needs to be extended by specific implementations. + """ + + def __init__(self, settings): + """ + Initialize the object by setting the instance variables, + decompressing the parsed service document and + preparing the auth handler. + """ + + # We expect to have the following instance variables: + # self.server_id (int) + # self.name (str) + # self.username (str) + # self.password (str) + # self.email (str) + # self.service_document_parsed (blob) + # self.update_frequency (str) + # self.last_updated (str/timestamp) + # self.realm (str) + # self.uri (str) + # self.service_document_url (str) + + for (name, value) in settings.iteritems(): + setattr(self, name, value) + + self._prepare_auth_handler() + + if self.service_document_parsed: + self.service_document_parsed = deserialize_via_marshal( + self.service_document_parsed + ) + else: + self.update() + + def _prepare_auth_handler(self): + """ + """ + + # create a password manager + self._password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() + + # Add the username and password + self._password_mgr.add_password(self.realm, + self.uri, + self.username, + self.password) + + def _url_opener(self): + """ + """ + + # Create a basic authentication handler + _auth_handler = urllib2.HTTPBasicAuthHandler(self._password_mgr) + + # Return a URL opener (OpenerDirector instance) + return urllib2.build_opener(_auth_handler) + + @staticmethod + def _get_extension_for_file_type(file_type): + + extensions = guess_all_extensions(file_type) + + if not extensions: + def _guess_extension_for_ambiguous_file_type(ambiguous_file_type): + return ambiguous_file_type.split('/')[-1] + extensions = CFG_BIBSWORD_FILE_TYPES_MAPPING.get( + file_type, + _guess_extension_for_ambiguous_file_type(file_type)) + + return extensions + + def submit(self, metadata, media, url): + """ + """ + + # Perform the media deposit + media_response = self._deposit_media( + metadata, + media, + url + ) + + # Perform the metadata submission + metadata_response = self._ingest_metadata( + metadata, + media_response, + url + ) + + # Prepare and return the final response + response = self._prepare_response( + media_response, + metadata_response + ) + + return response + + def _deposit_media(self, metadata, media, url): + """ + """ + + # Hold the response for each file in the response dictionary + response = {} + + prepared_media = self._prepare_media(media) + + for (file_index, file_info) in prepared_media.iteritems(): + headers = self._prepare_media_headers(file_info, metadata) + try: + file_object = open(file_info['path'], "r") + file_contents = file_object.read() + file_object.close() + except Exception, e: + response[file_index] = { + 'error': True, + 'msg': str(e), + 'name': file_info['name'], + 'mime': file_info['mime'], + } + else: + req = urllib2.Request(url, file_contents, headers) + try: + if CFG_BIBSWORD_FORCE_DEFAULT_TIMEOUT: + default_timeout = getdefaulttimeout() + setdefaulttimeout(CFG_BIBSWORD_DEFAULT_TIMEOUT) + res = self._url_opener().open(req) + if CFG_BIBSWORD_FORCE_DEFAULT_TIMEOUT: + setdefaulttimeout(default_timeout) + except urllib2.HTTPError, e: + response[file_index] = { + 'error': True, + 'msg': self._prepare_media_response_error(e), + 'name': file_info['name'], + 'mime': file_info['mime'], + } + except Exception, e: + response[file_index] = { + 'error': True, + 'msg': str(e), + 'name': file_info['name'], + 'mime': file_info['mime'], + } + else: + response[file_index] = { + 'error': False, + 'msg': self._prepare_media_response(res), + 'name': file_info['name'], + 'mime': file_info['mime'], + } + + return response + + def _prepare_media(self, media): + """ + """ + + # NOTE: Implement me for each server! + + return media + + def _prepare_media_headers(self, file_info, dummy): + """ + """ + + # NOTE: Implement me for each server! + + headers = {} + + headers['Content-Type'] = file_info['mime'] + headers['Content-Length'] = file_info['size'] + headers['User-Agent'] = CFG_BIBSWORD_USER_AGENT + + return headers + + def _prepare_media_response(self, response): + """ + """ + + # NOTE: Implement me for each server! + + return response.read() + + def _prepare_media_response_error(self, error): + """ + """ + + # NOTE: Implement me for each server! + + return error + + def _ingest_metadata(self, metadata, media_response, url): + """ + """ + + response = {} + + if media_response: + for file_response in media_response.itervalues(): + if file_response['error']: + response['error'] = True + response['msg'] = ( + "There has been at least one error with " + + "the files chosen for media deposit." + ) + return response + else: + response['error'] = True + response['msg'] = ( + "No files were chosen for media deposit. " + + "At least one file has to be chosen." + ) + return response + + prepared_metadata = self._prepare_metadata(metadata, media_response) + + headers = self._prepare_metadata_headers(metadata) + + req = urllib2.Request(url, prepared_metadata, headers) + try: + if CFG_BIBSWORD_FORCE_DEFAULT_TIMEOUT: + default_timeout = getdefaulttimeout() + setdefaulttimeout(CFG_BIBSWORD_DEFAULT_TIMEOUT) + res = self._url_opener().open(req) + if CFG_BIBSWORD_FORCE_DEFAULT_TIMEOUT: + setdefaulttimeout(default_timeout) + except urllib2.HTTPError, e: + response['error'] = True + response['msg'] = self._prepare_metadata_response_error(e) + except Exception, e: + response['error'] = True + response['msg'] = str(e) + else: + response['error'] = False + response['msg'] = self._prepare_metadata_response(res) + + return response + + def _prepare_metadata(self, metadata, dummy): + """ + """ + + # NOTE: Implement me for each server! + + return metadata + + def _prepare_metadata_headers(self, dummy): + """ + """ + + # NOTE: Implement me for each server! + + headers = {} + + headers['Content-Type'] = CFG_BIBSWORD_ATOM_ENTRY_MIME_TYPE + headers['User-Agent'] = CFG_BIBSWORD_USER_AGENT + + return headers + + def _prepare_metadata_response(self, response): + """ + """ + + # NOTE: Implement me for each server! + + return response.read() + + def _prepare_metadata_response_error(self, error): + """ + """ + + # NOTE: Implement me for each server! + + return error + + def _prepare_response(self, media_response, metadata_response): + """ + """ + + # NOTE: Implement me for each server! + + return None + + def status(self, url): + """ + """ + + response = {} + + headers = self._prepare_status_headers() + + req = urllib2.Request(url, headers=headers) + + try: + if CFG_BIBSWORD_FORCE_DEFAULT_TIMEOUT: + default_timeout = getdefaulttimeout() + setdefaulttimeout(CFG_BIBSWORD_DEFAULT_TIMEOUT) + # TODO: No need for authentication at this point, + # maybe use the standard url_opener? + res = self._url_opener().open(req) + if CFG_BIBSWORD_FORCE_DEFAULT_TIMEOUT: + setdefaulttimeout(default_timeout) + except urllib2.HTTPError, e: + response['error'] = True + response['msg'] = str(e) + except Exception, e: + response['error'] = True + response['msg'] = str(e) + else: + response['error'] = False + response['msg'] = self._prepare_status_response(res) + self._store_submission_status( + url, + response['msg']['error'] is None and "{0}".format( + response['msg']['status'] + ) or "{0} ({1})".format( + response['msg']['status'], + response['msg']['error'] + ) + ) + + return response + + def _prepare_status_headers(self): + """ + """ + + # NOTE: Implement me for each server! + + headers = {} + + headers['User-Agent'] = CFG_BIBSWORD_USER_AGENT + + return headers + + def _prepare_status_response(self, response): + """ + """ + + # NOTE: Implement me for each server! + + return response.read() + + def _store_submission_status(self, url, status): + """ + Stores the submission status in the database. + """ + + query = """ + UPDATE swrCLIENTSUBMISSION + SET status=%s, + last_updated=%s + WHERE id_server=%s + AND url_alternate=%s + """ + + params = ( + status, + convert_datestruct_to_datetext(datetime.now()), + self.server_id, + url, + ) + + result = run_sql(query, params) + + return result + + @staticmethod + def _convert_frequency_to_timedelta(raw_frequency): + """ + Converts this: "5w4d3h2m1s" + to this: datetime.timedelta(39, 10921) + """ + + frequency_translation = { + "w": "weeks", + "d": "days", + "h": "hours", + "m": "minutes", + "s": "seconds", + } + + frequency_parts = re.findall( + "([0-9]+)([wdhms]{1})", + raw_frequency + ) + + frequency_parts = map( + lambda p: (frequency_translation[p[1]], int(p[0])), + frequency_parts + ) + + frequency_timedelta_parts = dict(frequency_parts) + + frequency_timedelta = timedelta(**frequency_timedelta_parts) + + return frequency_timedelta + + def needs_to_be_updated(self): + """ + Check if the service document is out of date, + i.e. if the "last_updated" date is more than + "update_frequency" time in the past. + """ + + frequency_timedelta = \ + SwordClientServer._convert_frequency_to_timedelta( + self.update_frequency + ) + + if ((self.last_updated + frequency_timedelta) < datetime.now()): + return True + + return False + + def update(self): + """ + Fetches the latest service document, parses it and + saves it in the database. + """ + + service_document_raw = self._fetch_service_document() + + if service_document_raw: + service_document_parsed = self._parse_service_document( + service_document_raw + ) + + if service_document_parsed: + self.service_document_parsed = service_document_parsed + return self._store_service_document() + + return False + + def _fetch_service_document(self): + """ + Returns the raw service document of the server. + """ + + req = urllib2.Request(self.service_document_url) + try: + res = self._url_opener().open(req) + except urllib2.HTTPError: + return None + service_document_raw = res.read() + res.close() + + return service_document_raw + + def _store_service_document(self): + """ + Stores the compressed parsed service document in the database. + """ + + query = """ + UPDATE swrCLIENTSERVER + SET service_document_parsed=%s, + last_updated=%s + WHERE id=%s + """ + + params = ( + serialize_via_marshal(self.service_document_parsed), + convert_datestruct_to_datetext(datetime.now()), + self.server_id + ) + + result = run_sql(query, params) + + return result diff --git a/modules/bibsword/lib/bibsword_client_templates.py b/modules/bibsword/lib/bibsword_client_templates.py index b347dc927a..cc35ed9703 100644 --- a/modules/bibsword/lib/bibsword_client_templates.py +++ b/modules/bibsword/lib/bibsword_client_templates.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- - +# # This file is part of Invenio. -# Copyright (C) 2010, 2011 CERN. +# Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016 CERN. # # Invenio is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -17,1090 +17,2029 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. -''' -BibSWORD Client Templates -''' - -from invenio.config import CFG_SITE_URL, CFG_SITE_NAME, CFG_SITE_RECORD - -class BibSwordTemplate: - ''' - This class contains attributes and methods that allows to display all - information used by the BibSword web user interface. Theses informations - are form, validation or error messages - ''' - - def __init__(self): - ''' No init necessary for this class ''' - - #--------------------------------------------------------------------------- - # BibSword WebSubmit Interface - #--------------------------------------------------------------------------- - - def tmpl_display_submit_ack(self, remote_id, link): - ''' - This method generate the html code that displays the acknoledgement - message after the submission of a record. - @param remote_id: id of the record given by arXiv - @param link: links to modify or consult submission - @return: string containing the html code - ''' - - html = '' - - html += '''

    Success !

    ''' - html += '''

    The record has been successfully pushed to arXiv !
    ''' \ - '''You will get an email once it will be accepted by ''' \ - '''arXiv moderator.

    ''' - html += '''

    The arXiv id of the submission is: %s

    ''' % \ - remote_id - html += '''

    Manage your submission

    ''' +""" +BibSWORD Client Templates. +""" + +from cgi import escape +from datetime import datetime +import re +from invenio.config import( + CFG_SITE_RECORD, + CFG_SITE_URL +) +from invenio.messages import gettext_set_language + + +class TemplateSwordClient(object): + """ + Templates for the Sword client. + """ + + @staticmethod + def _tmpl_submission_table_row( + submission, + ln + ): + """ + Returns the HTML code for a server table row. + """ + (user, + record_id, + server_id, + server_name, + submitted, + status, + last_updated, + status_url + ) = submission + + table_row = """ + + """.format(record_id, server_id) + + table_row += """ + + {0} + + """.format(user) + + table_row += """ + + #{2} + + """.format(CFG_SITE_URL, CFG_SITE_RECORD, record_id) + + table_row += """ + + {0} + + """.format(server_name) + + table_row += """ + + {0} + + """.format( + TemplateSwordClient._humanize_datetime( + submitted, + ln + ) + ) + + table_row += """ + + {0} + + """.format(status) + + table_row += """ + + {0} + + """.format( + TemplateSwordClient._humanize_datetime( + last_updated, + ln + ) + ) + + # ↺ --> ↺ + html_options = """ + ↺ + """ + + table_row += """ + + {0} + + """.format(html_options.format(server_id, status_url, ln)) + + table_row += """ + + """ + + return table_row + + def tmpl_submissions( + self, + submissions, + ln + ): + """ + Returns the submissions table with all information and options. + """ + + _ = gettext_set_language(ln) + + html = """ + + + + + + + + + + + + + + + + {tbody} + + + +
    + {user_header_label} + + {record_header_label} + + {server_header_label} + + {submitted_header_label} + + {status_header_label} + + {updated_header_label} + + {options_header_label} +
    + """ + + html_tbody = "" + + for submission in submissions: + html_tbody += TemplateSwordClient._tmpl_submission_table_row( + submission, + ln + ) + + return html.format( + error_message=_("An error has occured. " + + "The administrators have been informed."), + user_header_label=_("User"), + record_header_label=_("Record"), + server_header_label=_("Server"), + submitted_header_label=_("Submitted"), + status_header_label=_("Status"), + updated_header_label=_("Last updated"), + options_header_label=_("Options"), + tbody=html_tbody + ) + + @staticmethod + def _tmpl_server_table_row( + server, + ln + ): + """ + Returns the HTML code for a server table row. + """ + (server_id, + name, + engine, + username, + dummy, + email, + dummy, + update_frequency, + last_updated) = server + + table_row = """ + + """.format(server_id) + + table_row += """ + + {0} + + """.format(name) + + table_row += """ + + {0} + + """.format(engine) + + table_row += """ + + {0} + + """.format(username) + + table_row += """ + + {0} + + """.format(email) + + table_row += """ + + {0} + + """.format( + TemplateSwordClient._humanize_frequency( + update_frequency, + ln + ) + ) + + table_row += """ + + {0} + + """.format( + TemplateSwordClient._humanize_datetime( + last_updated, + ln + ) + ) + + # ↺ --> ↺ + # ✎ --> ✎ + # ✗ --> ✗ + html_options = """ + ↺ +   + ✎ +   + ✗ + """ + + table_row += """ + + {0} + + """.format(html_options.format(server_id, ln)) + + table_row += """ + + """ + + return table_row + + def tmpl_servers( + self, + servers, + ln + ): + """ + Returns the servers table with all information and available options. + """ + + _ = gettext_set_language(ln) + + # ⊕ --> ⊕ + html = """ + + +
    + + ⊕ {add_label} + +
    + + + + + + + + + + + + + + + {tbody} + + + +
    + {name_header_label} + + {engine_header_label} + + {username_header_label} + + {email_header_label} + + {frequency_header_label} + + {last_updated_header_label} + + {options_header_label} +
    + """ + + html_tbody = "" + + for server in servers: + html_tbody += TemplateSwordClient._tmpl_server_table_row( + server, + ln + ) + + return html.format( + confirm_delete_label=_( + "Are you sure you want to delete this server?" + ), + add_label=_("Add server"), + error_message=_("An error has occured. " + + "The administrators have been informed."), + name_header_label=_("Name"), + engine_header_label=_("Engine"), + username_header_label=_("Username"), + email_header_label=_("E-mail"), + frequency_header_label=_("Update frequency"), + last_updated_header_label=_("Last updated"), + options_header_label=_("Options"), + tbody=html_tbody, + ln=ln + ) + + def tmpl_add_or_modify_server( + self, + server, + available_engines, + ln + ): + """ + Returns the servers table with all information and available options. + """ + + _ = gettext_set_language(ln) + + if server: + (server_id, + name, + engine, + username, + password, + email, + dummy, + update_frequency, + dummy) = server + label = _("Modify server") + content = """ + + + + """.format(server_id) + controls = "" + else: + (server_id, + name, + engine, + username, + password, + email, + dummy, + update_frequency, + dummy) = ("",)*9 + label = _("Add server") + content = """ + + + """ + controls = "" + + html_input = """ +
    +
    + +
    +
    + {subtitle} +
    +
    +
    + +
    +
    + """ + + html_select_option = """ + + """ + + html_select = """ +
    +
    + +
    +
    + {subtitle} +
    +
    +
    + +
    +
    + """ + + content += html_input.format( + title=_("Server name"), + subtitle=_("A custom name for the server that you can then choose to submit to."), + input_type="text", + name="sword_client_server_name", + value=escape(name, True), + size="50%", + placeholder="example.com SWORD server" + ) + + content += html_select.format( + title=_("Server engine"), + subtitle=_("The server engine to connect to."), + name="sword_client_server_engine", + options="".join( + [html_select_option.format( + value=escape(available_engine, True), + selected=" selected" if available_engine == engine else "", + label=escape(available_engine, True) + ) for available_engine in available_engines] + ) + ) + + content += html_input.format( + title=_("Username"), + subtitle=_("The username used to connect to the server."), + input_type="text", + name="sword_client_server_username", + value=escape(username, True), + size="50%", + placeholder="johndoe" + ) + + content += html_input.format( + title=_("Password"), + subtitle=_("The password used to connect to the server."), + input_type="password", + name="sword_client_server_password", + value=password, + size="50%", + placeholder="pa55w0rd" + ) + + content += html_input.format( + title=_("E-mail"), + subtitle=_("The e-mail of the user that connects to the server."), + input_type="text", + name="sword_client_server_email", + value=escape(email, True), + size="50%", + placeholder="johndoe@example.com" + ) + + content += html_input.format( + title=_("Update frequency"), + subtitle=_("How often should the server be checked for updates?" + + " The update frequency must be expressed in units of" + + " weeks (w), days (d), hours(h), minutes (m) and" + + " seconds (s) as a single string with no spaces." + + " For example, \"1w3d\" means \"Every 1 week and 3" + + " days\" and \"6h30m10s\" means \"Every 6 hours," + + " 30 minutes and 10 seconds\"."), + input_type="text", + name="sword_client_server_update_frequency", + value=escape(update_frequency, True), + size="50%", + placeholder="1w3d" + ) + + controls += """ + + """.format( + label=_("Cancel") + ) + + controls += """ +   + """ + + controls += """ + + """.format( + label=_("Submit") + ) + + html = """ + + +
    +
    +
    + {label} +
    +
    +
    + {content} + +
    +
    + {controls} +
    +
    +
    + """.format( + label=label, + content=content, + controls=controls, + error_message=_("An error has occured. " + + "The administrators have been informed.") + ) return html - #--------------------------------------------------------------------------- - # BibSword Administrator Interface - #--------------------------------------------------------------------------- - - def tmpl_display_admin_page(self, submissions, first_row, last_row, - total_rows, is_prev, is_last, offset, - error_messages=None): - ''' - format the html code that display the submission table - @param submissions: list of all submissions and their status - @return: html code to be displayed - ''' - - if error_messages == None: - error_messages = [] - - body = ''' - - %(error_message)s - - - - - - -
    -
    -
    -
    - Display - - rows per page
    -
    - - - Pages %(first_row)s - %(last_row)s / %(total_rows)s - -
    - - - - - - - - - - - - - - %(submissions)s -
    -

    Submission state

    -
    Remote serverSubmitterRecord numberRemote idStatusDatesLinks
    -''' % { - 'error_message': \ - self.display_error_message_row(error_messages), - 'table_width' : '100%', - 'first_row' : first_row, - 'last_row' : last_row, - 'total_rows' : total_rows, - 'is_prev' : is_prev, - 'is_last' : is_last, - 'selected_1' : offset[0], - 'selected_2' : offset[1], - 'selected_3' : offset[2], - 'selected_4' : offset[3], - 'selected_5' : offset[4], - 'submissions' : self.fill_submission_table(submissions) - } - - return body - - - def tmpl_display_remote_server_info(self, server_info): - ''' - Display a table containing all server informations - @param server_info: tuple containing all server infos - @return: html code for the table containing infos - ''' - - body = '''\n''' \ - ''' \n''' \ - ''' \n''' \ - ''' \n''' \ - ''' \n ''' \ - ''' \n''' \ - ''' \n''' \ - ''' \n''' \ - ''' \n ''' \ - ''' \n''' \ - ''' \n''' \ - ''' \n''' \ - ''' \n ''' \ - ''' \n''' \ - ''' \n''' \ - ''' \n''' \ - ''' \n ''' \ - ''' \n''' \ - ''' \n''' \ - ''' \n''' \ - ''' \n ''' \ - ''' \n''' \ - ''' \n''' \ - ''' \n''' \ - ''' \n ''' \ - ''' \n''' \ - ''' \n''' \ - ''' \n''' \ - ''' \n ''' \ - ''' \n''' \ - ''' \n''' \ - ''' \n''' \ - ''' \n ''' \ - ''' \n''' \ - ''' \n'''\ - ''' \n''' \ - ''' \n ''' \ - '''
    ID%(server_id)s
    Name%(server_name)s
    Host%(server_host)s
    Username%(username)s
    Password%(password)s
    Email%(email)s
    Realm%(realm)s
    Record URL%(url_base_record)s
    URL Servicedocument%(url_servicedocument)s
    ''' % { - 'table_width' : '50%', - 'server_id' : server_info['server_id'], - 'server_name' : server_info['server_name'], - 'server_host' : server_info['server_host'], - 'username' : server_info['username'], - 'password' : server_info['password'], - 'email' : server_info['email'], - 'realm' : server_info['realm'], - 'url_base_record' : server_info['url_base_record'], - 'url_servicedocument': server_info['url_servicedocument'] - } - - return body - - - def tmpl_display_remote_servers(self, remote_servers, id_record, - error_messages): - ''' - format the html code that display a dropdown list containing the - servers - @param self: reference to the current instance of the class - @param remote_servers: list of tuple containing server's infos - @return: string containing html code - ''' - - body = ''' -
    - - %(error_message)s - - - - - - - - - - - - - - - - - -
    -

    Forward a record

    -
    -

    Enter the number of the report to submit:

    -     		- -
    -

    Select a remote server:

    -     		- -
    - -
    -
    ''' % { - 'error_message': \ - self.display_error_message_row(error_messages), - 'table_width' : '100%', - 'row_width' : '50%', - 'id_record' : id_record, - 'remote_server': \ - self.fill_dropdown_remote_servers(remote_servers) - } - - return body - - - def tmpl_display_collections(self, selected_server, server_infos, - collections, id_record, recid, error_messages): - ''' - format the html code that display the selected server, the informations - about the selected server and a dropdown list containing the server's - collections - @param self: reference to the current instance of the class - @param selected_server: tuple containing selected server name and id - @param server_infos: tuple containing infos about selected server - @param collections: list contianing server's collections - @return: string containing html code - ''' - - body = ''' -
    - - - - - - %(error_message)s - - - - - - - - - - - - - - - - -
    -

    Remote server

    -

    %(server_name)s

    -     		- SWORD version: %(server_version)s -
    - Max upload size [Kb]: %(server_maxUpload)s -
    - -
    -

    - - - - - - - - - - - - - -

    Collection

    -
    Select a collection: - -
    - -
    - -
    ''' % { - 'table_width' : '100%', - 'row_width' : '50%', - 'error_message' : \ - self.display_error_message_row(error_messages), - 'id_server' : selected_server['id'], - 'server_name' : selected_server['name'], - 'server_version' : server_infos['version'], - 'server_maxUpload': server_infos['maxUploadSize'], - 'collection' : \ - self.fill_dropdown_collections(collections), - 'id_record' : id_record, - 'recid' : recid - } - - return body - - - def tmpl_display_categories(self, selected_server, server_infos, - selected_collection, collection_infos, - primary_categories, secondary_categories, - id_record, recid, error_messages): - ''' - format the html code that display the selected server, the informations - about the selected server, the selected collections, the informations - about the collection and a dropdown list containing the server's - primary and secondary categories - @param self: reference to the current instance of the class - @param selected_server: tuple containing selected server name and id - @param server_infos: tuple containing infos about selected server - @param selected_collection: selected collection - @param collection_infos: tuple containing infos about selected col - @param primary_categories: list of mandated categories for the col - @return: string containing html code - ''' - - body = ''' -
    - - - - - - - %(error_message)s - - - - - - - - - - - - - - - - -
    -

    Remote server

    -
    -

    %(server_name)s

    -     		- SWORD version: %(server_version)s -
    - Max upload size [Kb]: %(server_maxUpload)s -
    - -
    -

    - - - - - - - - - - - - - - - - -
    -

    Collection

    -
    -

    %(collection_name)s

    -     		- URL: %(collection_url)s -
    - Accepted media types: -
      %(collection_accept)s
    -
    - -
    -

    - - - - - - - - - - -
    -

    Mandatory category

    -
    -

    Select a mandated category:

    -     		- -
    -

    - - - - - - - - - -
    -

    Optional categories

    -
    -

    Select optional categories:

    -     		- -
    -

    - -
    - -
    - -
    ''' % { - 'table_width' : '100%', - 'row_width' : '50%', - 'error_message' : self.display_error_message_row( - error_messages), - - # hidden input - 'id_server' : selected_server['id'], - 'id_collection' : selected_collection['id'], - 'id_record' : id_record, - 'recid' : recid, - - # variables values - 'server_name' : selected_server['name'], - 'server_version' : server_infos['version'], - 'server_maxUpload' : server_infos['maxUploadSize'], - - 'collection_name' : selected_collection['label'], - - 'collection_accept': ''.join([ - '''
  • %(name)s
    • ''' % { - 'name': accept - } for accept in collection_infos['accept'] ]), - - 'collection_url' : selected_collection['url'], - 'primary_categories' : self.fill_dropdown_primary( - primary_categories), - - 'secondary_categories': self.fill_dropdown_secondary( - secondary_categories) - } - - return body - - - def tmpl_display_metadata(self, user, server, collection, primary, - categories, medias, metadata, id_record, recid, - error_messages): - ''' - format a string containing every informations before a submission - ''' - - - body = ''' -
    - - - - - - - - - %(error_message)s - - - - - - - - - - - - - -%(categories)s - - - -
    -

    Destination

    -
    -

    %(server_name)s

    -     		- Collection: %(collection_name)s ( %(collection_url)s ) -
    - Primary category: %(primary_name)s ( %(primary_url)s ) -
    - -
    -

    - - - - - - - - - - - - - - -
    -

    Submitter

    -
    Name:
    Email:
    -

    - - - - - - -

    Media

    %(medias)s%(media_help)s
    -

    - - - - - - - - - - - - - - - - - - -%(contributors)s -%(journal_refs)s -%(report_nos)s -

    Metadata

    Warning: modification(s) will not be saved on the %(CFG_SITE_NAME)s -

    Report Number*:

    Title*:

    		 -

    Summary*:

    		 - -
    - -

    The fields having a * are mandatory

    - -
    - -
    - -''' % { - 'table_width' : '100%', - 'row_width' : '25%', - 'error_message' : \ - self.display_error_message_row(error_messages), - 'CFG_SITE_NAME': CFG_SITE_NAME, - - # hidden input - 'id_server' : server['id'], - 'id_collection' : collection['id'], - 'id_primary' : primary['id'], - 'id_categories' : self.get_list_id_categories(categories), - 'id_record' : id_record, - 'recid' : recid, - - # variables values - 'server_name' : server['name'], - 'collection_name' : collection['label'], - 'collection_url' : collection['url'], - 'primary_name' : primary['label'], - 'primary_url' : primary['url'], - 'categories' : self.fill_optional_category_list(categories), - - #user - 'user_name' : user['nickname'], - 'user_email' : user['email'], - - # media - 'medias' : self.fill_media_list(medias, server['id']), - 'media_help' : self.fill_arxiv_help_message(), - - # metadata - 'id' : metadata['id'], - 'title' : metadata['title'], - 'summary' : metadata['summary'], - 'contributors' : self.fill_contributors_list( - metadata['contributors']), - 'journal_refs' : self.fill_journal_refs_list( - metadata['journal_refs']), - 'report_nos' : self.fill_report_nos_list( - metadata['report_nos']) - } - - return body - - - def tmpl_display_list_submission(self, submissions): - ''' - Display the data of submitted recods - ''' - - body = ''' - - - - - - - - - - - - - - - %(submissions)s -
    -

    Document successfully submitted !

    -
    Remote serverSubmitterRecord idRemote idStatusDatesLinks
    - Return -
    ''' % { - 'table_width' : '100%', - 'submissions' : self.fill_submission_table(submissions), - 'CFG_SITE_URL' : CFG_SITE_URL - } - - return body - - - #*************************************************************************** - # Private functions - #*************************************************************************** - - - def display_error_message_row(self, error_messages): - ''' - return a list of error_message in form of a bullet list - @param error_messages: list of error_messages to display - @return: html code that display list of errors - ''' - - # if no errors, return nothing - if len(error_messages) == 0: - return '' - - if len(error_messages) == 1: - # display a generic header message - body = ''' - - - -

    The following error was found:

    -
      -''' + @staticmethod + def _humanize_frequency( + raw_frequency, + ln + ): + """ + Converts this: "3w4d5h6m7s" + to this: "Every 3 weeks, 4 days, 5 hours, 6 minutes, 7 seconds" + """ + + _ = gettext_set_language(ln) + + frequency_translation = { + "w": _("week(s)"), + "d": _("day(s)"), + "h": _("hour(s)"), + "m": _("minute(s)"), + "s": _("second(s)"), + } + + humanized_frequency = _("Every") + humanized_frequency += " " + + frequency_parts = re.findall( + "([0-9]+)([wdhms]{1})", + raw_frequency + ) + + frequency_parts = map( + lambda p: "{0} {1}".format(p[0], frequency_translation[p[1]]), + frequency_parts + ) + + humanized_frequency += ", ".join(frequency_parts) + + return humanized_frequency + + @staticmethod + def _humanize_datetime( + raw_datetime, + ln + ): + """ + Converts this: "2015-02-17 10:10:10" + to this: "2 week(s), 5 day(s) ago" + """ + + _ = gettext_set_language(ln) + + if not raw_datetime: + return _("Never") + + passed_datetime = datetime.now() - raw_datetime + + passed_datetime_days = passed_datetime.days + + humanized_datetime_parts = [] + + if passed_datetime_days: + passed_datetime_weeks = passed_datetime_days / 7 + if passed_datetime_weeks: + humanized_datetime_parts.append( + "{0!s} {1}".format(passed_datetime_weeks, _("week(s)")) + ) + passed_datetime_days = passed_datetime_days % 7 + humanized_datetime_parts.append( + "{0!s} {1}".format(passed_datetime_days, _("day(s)")) + ) else: - # display a generic header message - body = ''' - - - -

      Following errors were found:

      -
        -''' - - # insert each error lines - for error_message in error_messages: - body = body + ''' -
      • %(error)s
        • ''' % { - 'error': error_message + passed_datetime_seconds = passed_datetime.seconds + passed_datetime_minutes = passed_datetime_seconds / 60 + passed_datetime_hours = passed_datetime_minutes / 60 + if passed_datetime_hours: + humanized_datetime_parts.append( + "{0!s} {1}".format(passed_datetime_hours, _("hour(s)")) + ) + passed_datetime_minutes = passed_datetime_minutes % 60 + if passed_datetime_minutes: + humanized_datetime_parts.append( + "{0!s} {1}".format(passed_datetime_minutes, _("minute(s)")) + ) + passed_datetime_seconds = passed_datetime_seconds % 60 + if passed_datetime_seconds: + humanized_datetime_parts.append( + "{0!s} {1}".format(passed_datetime_seconds, _("second(s)")) + ) + + humanized_datetime = ", ".join(humanized_datetime_parts) + humanized_datetime += " " + humanized_datetime += _("ago") + + return humanized_datetime + + @staticmethod + def _submit_prepare_select_tag( + select_id, + select_name, + options, + default_option=None, + multiple=False + ): + """Prepare the HTML select tag.""" + out = """ + + """ + + return out + + def tmpl_submit( + self, + sid, + recid, + title, + author, + report_number, + step, + servers, + server_id, + ln + ): + """ + The main submission interface. + + Through a series of steps, the user is guided through the submission. + """ + + _ = gettext_set_language(ln) + + out = """ +
        + +
        + %(header)s +
        + +
        + %(step_1_title)s +
        + +
        + %(step_1_details)s +
        + + + + + + + + + + + + + + + + + +
        + + + + + + """ % { + "header": self._submit_header( + recid, + title, + author, + report_number, + ln + ), + + "step_1_title": _("Where would you like to submit?"), + "step_1_details": self.submit_step_1_details( + servers, + server_id, + ln + ), + "step_1_title_done_text": _("Selected server:"), + "step_1_server_select_id": "bibsword_submit_step_1_select", + "step_1_server_confirm_id": "bibsword_submit_step_1_confirm", + + "step_2_title": _("What collection would you like to submit to?"), + "step_2_details": self._submit_loading(ln), + "step_2_title_done_text": _("Selected collection:"), + "step_2_collection_select_id": "bibsword_submit_step_2_select", + "step_2_collection_confirm_id": "bibsword_submit_step_2_confirm", + + "step_3_title": _("What category would you like to submit to?"), + "step_3_details": self._submit_loading(ln), + "step_3_optional_categories_add_id": + "bibsword_submit_step_3_optional_categories_add", + "step_3_optional_categories_select_id": + "bibsword_submit_step_3_optional_categories_select", + "step_3_optional_categories_legend_id": + "bibsword_submit_step_3_optional_categories_legend", + "step_3_optional_categories_remove_image": + "%s/%s" % (CFG_SITE_URL, "img/cross_red.gif"), + "step_3_title_done_text_part_1": _("Selected category:"), + "step_3_title_done_text_part_2": _("additional categories"), + "step_3_mandatory_category_select_id": + "bibsword_submit_step_3_mandatory_category_select", + "step_3_categories_confirm_id": + "bibsword_submit_step_3_collection_confirm", + + "step_4_additional_rn_insert_class": + "bibsword_submit_step_4_additional_rn_insert", + "step_4_additional_rn_insert_data": + "".join(TemplateSwordClient._submit_step_4_additional_rn_input_block( + name='additional_rn', + value='', + size='25', + placeholder=_('Report number...'), + sortable_label=_("⇳"), + remove_class='bibsword_submit_step_4_additional_rn_remove negative', + remove_label=_("✘") + ).splitlines()), + "step_4_additional_rn_remove_class": + "bibsword_submit_step_4_additional_rn_remove", + "step_4_contributors_insert_class": + "bibsword_submit_step_4_contributors_insert", + "step_4_contributors_insert_data": + "".join(TemplateSwordClient._submit_step_4_contributors_input_block( + fullname=( + _('Fullname:'), + 'contributor_fullname', + '', + '25', + _('Fullname...'), + ), + email=( + _('E-mail:'), + 'contributor_email', + '', + '35', + _('E-mail...'), + ), + affiliation=( + _('Affiliation:'), + 'contributor_affiliation', + '', + '35', + _('Affiliation...'), + ), + sortable_label=_("⇳"), + remove_class='bibsword_submit_step_4_contributors_remove negative', + remove_label=_("✘") + ).splitlines()), + "step_4_contributors_remove_class": + "bibsword_submit_step_4_contributors_remove", + "step_4_title": + _("Please review and, if needed," + + " correct the following information:"), + "step_4_details": self._submit_loading(ln), + "step_4_title_done_text": _( + "All the information has been prepared for submission" + ), + "step_4_submission_data_id": "bibsword_submit_step_4_submission_data", + "step_4_submission_confirm_id": "bibsword_submit_step_4_submission_confirm", + + "step_final_title": _( + "Please wait while your submission is being processed..." + ), + "step_final_details": self._submit_loading(ln), + "step_final_title_done_text": _( + "Your submission has been processed" + ), + + "title_done_color": "#999999", + "title_done_image": "%s/%s" % (CFG_SITE_URL, "img/aid_check.png"), + "animation_speed": 300, + "error_message": _("An error has occured. " + + "The administrators have been informed."), + + "CFG_SITE_URL": CFG_SITE_URL, + "sid": sid, + "step": step, + "ln": ln, + } - return html + return out + + def _submit_header( + self, + record_id, + title, + author, + report_number, + ln + ): + """ + """ + + _ = gettext_set_language(ln) + + out = """ + {submitting} {title} ({report_number}) {by} {author} + """.format( + submitting=_("You are submitting:"), + CFG_SITE_URL=CFG_SITE_URL, + CFG_SITE_RECORD=CFG_SITE_RECORD, + record_id=record_id, + title=escape(title, True), + report_number=escape(report_number, True), + by=_("by"), + author=escape(author, True) + ) + + return out + + def _submit_loading(self, ln): + """ + NOTE_2015 + """ + + _ = gettext_set_language(ln) + + out = """ +   %s + """ % (CFG_SITE_URL, "img/loading.gif", _("Loading...")) + return out + + def submit_step_1_details( + self, + servers, + server_id, + ln + ): + """ + """ + + _ = gettext_set_language(ln) + + confirmation = """ + + """ % ( + "bibsword_submit_step_1_confirm", + _("Confirm and continue") + ) + + out = """ +
        +
        + +
        +
        + %(selection)s +
        +
        +
        +
        + %(confirmation)s +
        +
        + """ % { + 'label': _("Server:"), + 'selection': TemplateSwordClient._submit_prepare_select_tag( + "bibsword_submit_step_1_select", + "server_id", + servers, + default_option=server_id + ), + 'confirmation': confirmation, + } + return out + + def submit_step_2_details( + self, + collections, + ln + ): + """ + NOTE_2015 + """ + + _ = gettext_set_language(ln) + + confirmation = """ + + """ % ( + "bibsword_submit_step_2_confirm", + _("Confirm and continue") + ) + + out = """ +
        +
        + +
        +
        + %(selection)s +
        +
        +
        +
        + %(confirmation)s +
        +
        + """ % { + "label": _("Collection:"), + "selection": TemplateSwordClient._submit_prepare_select_tag( + "bibsword_submit_step_2_select", + "collection_url", + collections + ), + "confirmation": confirmation + } - def get_list_id_categories(self, categories): - ''' - gives the id of the categores tuple - ''' + return out + + def submit_step_3_details( + self, + mandatory_categories, + optional_categories, + ln + ): + """ + """ + + _ = gettext_set_language(ln) + + confirmation = """ + + """ % ( + "bibsword_submit_step_3_collection_confirm", + _("Confirm and continue") + ) + + # TODO: Different SWORD servers will have different concepts + # of categories (mandatory, optional, etc). Write a function + # that accomodates all cases. + if not mandatory_categories: + mandatory_categories = optional_categories + optional_categories = None + + out = """ +
        +
        + +
        +
        + %(primary_category_selection)s +
        +
        + """ % { + 'label': _("Category:"), + 'primary_category_selection': + TemplateSwordClient._submit_prepare_select_tag( + "bibsword_submit_step_3_mandatory_category_select", + "mandatory_category_url", + mandatory_categories + ), + } - id_categories = [] + if optional_categories: + + addition = """ + + """ % ( + "bibsword_submit_step_3_optional_categories_add", + "" + _("✚") + "" + ) + + out += """ +
        +
        + +
        +
        + %(additional_category_selection)s + %(addition)s +
        +
        +
          +
        +
        +
        + """ % { + "label": _("Additional optional categories:"), + "additional_category_selection": + TemplateSwordClient._submit_prepare_select_tag( + "bibsword_submit_step_3_optional_categories_select", + None, + optional_categories + ), + "addition": addition, + "additional_category_legend_id": + "bibsword_submit_step_3_optional_categories_legend", + } + + out += """ +
        +
        + %(confirmation)s +
        +
        + """ % { + "confirmation": confirmation, + } - for category in categories: - id_categories.append(category['id']) + return out + + @staticmethod + def _submit_step_4_additional_rn_input_block( + name, + value, + size, + placeholder, + sortable_label, + remove_class, + remove_label + ): + """ + """ + + out = """ +
        +
        + %(sortable_label)s + + +
        +
        + """ % { + 'name': name, + 'value': value, + 'size': size, + 'placeholder': placeholder, + 'sortable_label': sortable_label, + 'remove_class': remove_class, + 'remove_label': remove_label, + } - return id_categories + return out + + @staticmethod + def _submit_step_4_contributors_input_block( + fullname, + email, + affiliation, + sortable_label, + remove_class, + remove_label + ): + """ + """ + + out = """ +
        +
        + %(sortable_label)s +
        +
        +
        + +
        +
        + +
        +
        +
        +
        + +
        +
        + +
        +
        +
        +
        + +
        +
        + +
        +
        +
        + +
        +
        + """ % { + 'label_fullname': fullname[0], + 'name_fullname': fullname[1], + 'value_fullname': fullname[2], + 'size_fullname': fullname[3], + 'placeholder_fullname': fullname[4], + + 'label_email': email[0], + 'name_email': email[1], + 'value_email': email[2], + 'size_email': email[3], + 'placeholder_email': email[4], + + 'label_affiliation': affiliation[0], + 'name_affiliation': affiliation[1], + 'value_affiliation': affiliation[2], + 'size_affiliation': affiliation[3], + 'placeholder_affiliation': affiliation[4], + + 'sortable_label': sortable_label, + 'remove_class': remove_class, + 'remove_label': remove_label, + } + return out + + def submit_step_4_details( + self, + metadata, + files, + maximum_number_of_contributors, + ln + ): + """ + """ + + _ = gettext_set_language(ln) + + ####################################################################### + # Confirmation button + ####################################################################### + confirmation_text = """ +
        +
        + +
        +
        + """ % { + 'id': "bibsword_submit_step_4_submission_confirm", + 'value': _("Confirm and submit"), + } - def format_media_list_by_type(self, medias): - ''' - This function format the media by type (Main, Uploaded, ...) - ''' + ####################################################################### + # Title + ####################################################################### + title_text = """ +
        +
        + +
        +
        + +
        +
        + """ % { + 'label': _('Title:'), + 'placeholder': _('Title...'), + 'value': metadata['title'] and escape(metadata['title'], True) or '', + 'size': '100', + 'name': 'title', + } - #format media list by type of document - media_type = [] + ####################################################################### + # Author + ####################################################################### + author_text = """ +
        +
        + +
        +
        +
        + +
        +
        + +
        +
        +
        +
        + +
        +
        + +
        +
        +
        +
        + +
        +
        + +
        +
        +
        + """ % { + 'label': _('Author:'), + 'label_fullname': _('Fullname:'), + 'placeholder_fullname': _('Fullname...'), + 'value_fullname': metadata['author'][0] and escape(metadata['author'][0], True) or '', + 'size_fullname': '25', + 'name_fullname': 'author_fullname', + 'label_email': _('E-mail:'), + 'placeholder_email': _('E-mail...'), + 'value_email': metadata['author'][1] and escape(metadata['author'][1], True) or '', + 'size_email': '35', + 'name_email': 'author_email', + 'label_affiliation': _('Affiliation:'), + 'placeholder_affiliation': _('Affiliation...'), + 'value_affiliation': metadata['author'][2] and escape(metadata['author'][2], True) or '', + 'size_affiliation': '35', + 'name_affiliation': 'author_affiliation', + } - for media in medias: + ####################################################################### + # Contributors + ####################################################################### + contributors_text = """ +
        +
        + +
        + """ % { + 'label': _('Contributors:'), + } - # if it is the first media of this type, create a new type - is_type_in_media_type = False - for type in media_type: - if media['collection'] == type['media_type']: - is_type_in_media_type = True + if len(metadata['contributors']) > maximum_number_of_contributors: + contributors_text += """ +
        + %(text)s +
        + """ % { + 'text': _('Displaying only the first {0}').format(maximum_number_of_contributors), + } + + contributors_text += """ +
        + """ + + for contributor in metadata['contributors'][:maximum_number_of_contributors]: + + contributors_text += TemplateSwordClient._submit_step_4_contributors_input_block( + fullname=( + _('Fullname:'), + 'contributor_fullname', + contributor[0] and escape(contributor[0], True) or '', + '25', + _('Fullname...'), + ), + email=( + _('E-mail:'), + 'contributor_email', + contributor[1] and escape(contributor[1], True) or '', + '35', + _('E-mail...'), + ), + affiliation=( + _('Affiliation:'), + 'contributor_affiliation', + contributor[2] and escape(contributor[2], True) or '', + '35', + _('Affiliation...'), + ), + sortable_label=_("⇳"), + remove_class='bibsword_submit_step_4_contributors_remove negative', + remove_label=_("✘") + ) + + contributors_text += """ +
        + """ + + contributors_text += """ +
        +
        + + +
        +
        + """ % { + 'insert_class': 'bibsword_submit_step_4_contributors_insert positive', + 'insert_label': _("✚"), + 'label': _("Insert another one..."), + } - if is_type_in_media_type == False: - type = {} - type['media_type'] = media['collection'] - type['media_list'] = [] - media_type.append(type) + contributors_text += """ +
        + """ + + ####################################################################### + # Abstract + ####################################################################### + abstract_text = """ +
        +
        + +
        +
        + +
        +
        + """ % { + 'label': _('Abstract:'), + 'value': metadata['abstract'] and escape(metadata['abstract'], True) or '', + 'rows': '10', + 'cols': '120', + 'name': 'abstract', + } + + ####################################################################### + # Report number + ####################################################################### + rn_text = """ +
        +
        + +
        +
        + +
        +
        + """ % { + 'label': _('Report number:'), + 'placeholder': _('Report number...'), + 'value': metadata['rn'] and escape(metadata['rn'], True) or '', + 'size': '25', + 'name': 'rn', + } + + ####################################################################### + # Additional report numbers + ####################################################################### + additional_rn_text = """ +
        +
        + +
        + """ % { + 'label': _('Additional report numbers:'), + } - # insert the media in the good media_type element - for type in media_type: - if type['media_type'] == media['collection']: - type['media_list'].append(media) + additional_rn_text += """ +
        + """ + + for additional_rn in metadata['additional_rn']: + + additional_rn_text += TemplateSwordClient._submit_step_4_additional_rn_input_block( + name='additional_rn', + value=additional_rn and escape(additional_rn, True) or '', + size='25', + placeholder=_('Report number...'), + sortable_label=_("⇳"), + remove_class='bibsword_submit_step_4_additional_rn_remove negative', + remove_label=_("✘") + ) + + additional_rn_text += """ +
        + """ + + additional_rn_text += """ +
        +
        + + +
        +
        + """ % { + 'insert_class': 'bibsword_submit_step_4_additional_rn_insert positive', + 'insert_label': _("✚"), + 'label': _("Insert another one..."), + } + + additional_rn_text += """ +
        + """ + + ####################################################################### + # DOI + ####################################################################### + # if metadata['doi']: + + # doi_text = """ + # + # """ % { + # 'name': 'doi', + # 'value': escape(metadata['doi'], True), + # } + + # else: + # doi_text = '' + + ####################################################################### + # Journal information (code, title, page, year) + ####################################################################### + # journal_info_text = '' + + # for journal_info in zip(('code', 'title', 'page', 'year'), metadata['journal_info']): + + # if journal_info[1]: + # journal_info_text += """ + # + # """ % { + # 'name': journal_info[0], + # 'value': escape(journal_info[1], True), + # } + + ####################################################################### + # Files + ####################################################################### + if files: + files_text = """ +
        +
        + +
        + """ % { + 'label': _('Files'), + } + + for file_key in files.iterkeys(): + + files_text += """ +
        + + %(label)s +
        + """ % { + 'name': 'files', + 'value': str(file_key), + 'url': escape(files[file_key]['url'], True), + 'label': escape(files[file_key]['name'], True), + } + + files_text += """ +
        + """ + + else: + files_text = "" + + ####################################################################### + # Complete final text. Reorder if necessary. + ####################################################################### + text_open = """ +
        + """ % { + "id": "bibsword_submit_step_4_submission_data", + } - return media_type + text_body = ( + rn_text + + additional_rn_text + + title_text + + author_text + + abstract_text + + contributors_text + + files_text + + confirmation_text + ) + + text_close = """ +
        + """ + + return text_open + text_body + text_close + + def submit_step_final_details( + self, + ln + ): + """ + Return the HTML for the final step details. + """ + + _ = gettext_set_language(ln) + + out = _("The submission has been completed successfully.
        " + + "You may check the status of all your submissions " + + "here.").format(ln) + + return out diff --git a/modules/bibsword/lib/bibsword_client_tester.py b/modules/bibsword/lib/bibsword_client_tester.py deleted file mode 100644 index 778d6fe5ef..0000000000 --- a/modules/bibsword/lib/bibsword_client_tester.py +++ /dev/null @@ -1,668 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of Invenio. -# Copyright (C) 2010, 2011 CERN. -# -# Invenio is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of the -# License, or (at your option) any later version. -# -# Invenio is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Invenio; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. - -"""Test cases for the BibSword module.""" - -__revision__ = "$Id$" - -# pylint: disable-msg=C0301 - -from invenio.testutils import InvenioTestCase -import os -import sys -import time -from invenio.testutils import make_test_suite, run_test_suite -from invenio.config import CFG_TMPDIR -from invenio.bibsword_client_formatter import format_marcxml_file, \ - format_submission_status, \ - ArXivFormat -from invenio.bibsword_client import format_metadata, \ - list_submitted_resources, \ - perform_submission_process - -from invenio.bibsword_client_http import RemoteSwordServer -from invenio.bibsword_client_dblayer import get_remote_server_auth, \ - insert_into_swr_clientdata, \ - update_submission_status, \ - select_submitted_record_infos, \ - delete_from_swr_clientdata -from invenio.dbquery import run_sql -from invenio.bibsword_config import CFG_SUBMISSION_STATUS_SUBMITTED, \ - CFG_SUBMISSION_STATUS_PUBLISHED, \ - CFG_SUBMISSION_STATUS_REMOVED -from xml.dom import minidom - -class Test_format_marcxml_file(InvenioTestCase): - """ bibsword_format - test the parsing and extracting of marcxml nodes""" - - def test_extract_marcxml_1(self): - """Test_format_marcxml_file - extract marcxml without id, report_nos and comment""" - # Test with marcxml file 1 - marcxml = open("%s%sTest_marcxml_file_1.xml" % (CFG_TMPDIR, os.sep)).read() - metadata = format_marcxml_file(marcxml) - self.assertEqual(metadata['id'], '') - self.assertEqual(metadata['title'], "Calorimetry triggering in ATLAS") - self.assertEqual(metadata['contributors'][0]['name'], "Igonkina, O") - self.assertEqual(metadata['contributors'][0]['affiliation'][0], "NIKHEF, Amsterdam") - self.assertEqual(metadata['summary'], "The ATLAS experiment is preparing for data taking at 14 TeV collision energy. A rich discovery physics program is being prepared in addition to the detailed study of Standard Model processes which will be produced in abundance. The ATLAS multi-level trigger system is designed to accept one event in 2 105 to enable the selection of rare and unusual physics events. The ATLAS calorimeter system is a precise instrument, which includes liquid Argon electro-magnetic and hadronic components as well as a scintillator-tile hadronic calorimeter. All these components are used in the various levels of the trigger system. A wide physics coverage is ensured by inclusively selecting events with candidate electrons, photons, taus, jets or those with large missing transverse energy. The commissioning of the trigger system is being performed with cosmic ray events and by replaying simulated Monte Carlo events through the trigger and data acquisition system.") - self.assertEqual(metadata['contributors'][1]['name'], "Achenbach, R") - self.assertEqual(metadata['contributors'][1]['affiliation'][0], "Kirchhoff Inst. Phys.") - self.assertEqual(metadata['contributors'][2]['name'], "Adragna, P") - self.assertEqual(metadata['contributors'][2]['affiliation'][0], "Queen Mary, U. of London") - nb_contributors = len(metadata['contributors']) - self.assertEqual(nb_contributors, 205) - self.assertEqual(metadata['contributors'][204]['name'], "Özcan, E") - self.assertEqual(metadata['contributors'][204]['affiliation'][0], "University Coll. London") - self.assertEqual(metadata['doi'], "10.1088/1742-6596/160/1/012061") - self.assertEqual(metadata['journal_refs'][0], "J. Phys.: Conf. Ser.: 012061 (2009) pp. 160") - self.assertEqual(len(metadata['journal_refs']), 1) - self.assertEqual('report_nos' in metadata, True) - self.assertEqual(metadata['comment'], '') - - - def test_extract_marcxml_2(self): - """Test_format_marcxml_file - extract marcxml without report_nos, doi""" - - #Test with marcxml file 2 - marcxml = open("%s%sTest_marcxml_file_2.xml" % (CFG_TMPDIR, os.sep)).read() - metadata = format_marcxml_file(marcxml) - self.assertEqual(metadata['id'], "arXiv:1001.1674") - self.assertEqual(metadata['title'], "Persistent storage of non-event data in the CMS databases") - self.assertEqual(metadata['contributors'][0]['name'], "De Gruttola, M") - self.assertEqual(metadata['contributors'][0]['affiliation'][0], "CERN") - self.assertEqual(metadata['contributors'][0]['affiliation'][1], "INFN, Naples") - self.assertEqual(metadata['contributors'][0]['affiliation'][2], "Naples U.") - self.assertEqual(metadata['summary'], "In the CMS experiment, the non event data needed to set up the detector, or being produced by it, and needed to calibrate the physical responses of the detector itself are stored in ORACLE databases. The large amount of data to be stored, the number of clients involved and the performance requirements make the database system an essential service for the experiment to run. This note describes the CMS condition database architecture, the data-flow and PopCon, the tool built in order to populate the offline databases. Finally, the first results obtained during the 2008 and 2009 cosmic data taking are presented.") - self.assertEqual(metadata['contributors'][1]['name'], "Di Guida, S") - self.assertEqual(metadata['contributors'][1]['affiliation'][0], "CERN") - self.assertEqual(metadata['contributors'][2]['name'], "Futyan, D") - self.assertEqual(metadata['contributors'][2]['affiliation'][0], "Imperial Coll., London") - nb_contributors = len(metadata['contributors']) - self.assertEqual(nb_contributors, 11) - self.assertEqual(metadata['contributors'][10]['name'], "Xie, Z") - self.assertEqual(metadata['contributors'][10]['affiliation'][0], "Princeton U.") - self.assertEqual(metadata['doi'], '') - self.assertEqual(metadata['journal_refs'][0], "JINST: P04003 (2010) pp. 5") - self.assertEqual(len(metadata['journal_refs']), 1) - self.assertEqual('report_nos' in metadata, True) - self.assertEqual(metadata['comment'], "Comments: 20 pages, submitted to IOP") - - def test_extract_full_marcxml_3(self): - """Test_format_marcxml_file - extract marcxml without doi""" - - #Test with marcxml file 3 - marcxml = open("%s%sTest_marcxml_file_3.xml" % (CFG_TMPDIR, os.sep)).read() - metadata = format_marcxml_file(marcxml) - self.assertEqual(metadata['id'], "ATL-PHYS-CONF-2007-008") - self.assertEqual(metadata['title'], "Early Standard Model physics and early discovery strategy in ATLAS") - self.assertEqual(metadata['contributors'][0]['name'], "Grosse-Knetter, J") - self.assertEqual(metadata['contributors'][0]['affiliation'][0], "Bonn U.") - self.assertEqual(metadata['summary'], "In 2008 the LHC will open a new energy domain for physics within the Standard Model and beyond. The physics channels which will be addressed by the ATLAS experiment in the initial period of operation will be discussed. These include Standard Model processes such as W/Z production and early top measurements. This will be followed by a description of the searches for a low-mass Higgs boson, new heavy di-lepton resonances, and Supersymmetry, for which a striking signal might be observed after only a few months of data taking.") - self.assertEqual(len(metadata['contributors']), 1) - self.assertEqual(metadata['doi'], '') - self.assertEqual(metadata['journal_refs'][0], "Nucl. Phys. B, Proc. Suppl.: 55-59 (2008) pp. 177-178") - self.assertEqual(len(metadata['journal_refs']), 1) - self.assertEqual(len(metadata['report_nos']), 2) - self.assertEqual(metadata['report_nos'][0], "ATL-COM-PHYS-2007-036") - self.assertEqual(metadata['report_nos'][1], "CERN-ATL-COM-PHYS-2007-036") - self.assertEqual(metadata['comment'], '') - - def test_extract_null_marcxml(self): - """Test_format_marcxml_file - no metadata""" - - #Test without any marcxml file - metadata = format_marcxml_file("") - self.assertEqual(metadata["error"], "MARCXML string is empty !") - - def test_extract_wrong_file(self): - """Test_format_marcxml_file - unexistant metadata file""" - - #Test with a unknown marcxml file - metadata = format_marcxml_file("%s%sTest_marcxml_file_false.xml" % (CFG_TMPDIR, os.sep), True) - self.assertEqual(metadata["error"], "Unable to open marcxml file !") - - -class Test_format_metadata(InvenioTestCase): - """ bibsword - test the collection of all metadata """ - - def test_correct_metadata_collection(self): - """Test_format_metadata - collection of metadata without errors""" - - marcxml = open("%s%sTest_marcxml_file_3.xml" % (CFG_TMPDIR, os.sep)).read() - - metadata = {} - metadata['primary_label'] = 'Test - Test Disruptive Networks' - metadata['primary_url'] = 'http://arxiv.org/terms/arXiv/test.dis-nn' - - user_info = {} - user_info['nickname'] = 'test_user' - user_info['email'] = 'test@user.com' - - deposit_results = [] - deposit_results.append(open("%s%sTest_media_deposit.xml" % (CFG_TMPDIR, os.sep)).read()) - metadata = format_metadata(marcxml, deposit_results, user_info, metadata) - - self.assertEqual(metadata['id'], "ATL-PHYS-CONF-2007-008") - self.assertEqual(metadata['title'], "Early Standard Model physics and early discovery strategy in ATLAS") - self.assertEqual(metadata['contributors'][0]['name'], "Grosse-Knetter, J") - self.assertEqual(metadata['contributors'][0]['affiliation'][0], "Bonn U.") - self.assertEqual(metadata['summary'], "In 2008 the LHC will open a new energy domain for physics within the Standard Model and beyond. The physics channels which will be addressed by the ATLAS experiment in the initial period of operation will be discussed. These include Standard Model processes such as W/Z production and early top measurements. This will be followed by a description of the searches for a low-mass Higgs boson, new heavy di-lepton resonances, and Supersymmetry, for which a striking signal might be observed after only a few months of data taking.") - nb_contributors = len(metadata['contributors']) - self.assertEqual(nb_contributors, 1) - self.assertEqual(metadata['doi'], "") - self.assertEqual(len(metadata['journal_refs']), 1) - self.assertEqual(metadata['journal_refs'][0], "Nucl. Phys. B, Proc. Suppl.: 55-59 (2008) pp. 177-178") - self.assertEqual(len(metadata['report_nos']), 2) - self.assertEqual(metadata['report_nos'][0], "ATL-COM-PHYS-2007-036") - self.assertEqual(metadata['report_nos'][1], "CERN-ATL-COM-PHYS-2007-036") - self.assertEqual(metadata['comment'], "") - - self.assertEqual(metadata['primary_label'], "Test - Test Disruptive Networks") - self.assertEqual(metadata['primary_url'], "http://arxiv.org/terms/arXiv/test.dis-nn") - - self.assertEqual(metadata['author_name'], "test_user") - self.assertEqual(metadata['author_email'], "test@user.com") - - self.assertEqual(metadata['links']['link'], "https://arxiv.org/sword-app/edit/10070072") - self.assertEqual(metadata['links']['type'], "application/pdf") - - - def test_metadata_collection_no_data(self): - """Test_format_metadata - collection of metadata without any changes""" - - # Gives an empty marcxml file - marcxml = "" - # Gives no metadata - metadata = {} - # Gives no user informations - user_info = {} - # Gives no result where to find a link - deposit_results = [] - - metadata = format_metadata(marcxml, deposit_results, user_info, metadata) - self.assertEquals(len(metadata['error']), 9) - self.assertEquals(metadata['error'][0], "No submitter name given !") - self.assertEquals(metadata['error'][1], "No submitter email given !") - self.assertEquals(metadata['error'][2], "No primary category label given !") - self.assertEquals(metadata['error'][3], "No primary category url given !") - self.assertEquals(metadata['error'][4], "No links to the media deposit found !") - self.assertEquals(metadata['error'][5], "No document id given !") - self.assertEquals(metadata['error'][6], "No title given !") - self.assertEquals(metadata['error'][7], "No author given !") - self.assertEquals(metadata['error'][8], "No summary given !") - - - self.assertEquals(metadata['id'], "") - self.assertEquals(metadata['title'], "") - self.assertEqual(len(metadata['contributors']), 0) - self.assertEqual(metadata['summary'], "") - self.assertEqual(metadata['doi'], "") - self.assertEqual(len(metadata['journal_refs']), 0) - self.assertEqual(len(metadata['report_nos']), 0) - self.assertEqual(metadata['comment'], "") - self.assertEqual(metadata['primary_label'], "") - self.assertEqual(metadata['primary_url'], "") - self.assertEqual(metadata['author_name'], "") - self.assertEqual(metadata['author_email'], "") - self.assertEqual(len(metadata['links']), 0) - - -class Test_get_submission_status(InvenioTestCase): - """ bibsword_httpquery - test the get_submission_status method """ - - def test_get_submission_status_ok(self): - """Test_get_submission_status - connect to an existing url""" - - authentication_info = get_remote_server_auth(4) - connection = RemoteSwordServer(authentication_info) - result = connection.get_submission_status("http://arxiv.org/resolve/app/10070073") - - self.assertEqual(result != "", True) - - - def test_get_submission_status_no_url(self): - """Test_get_submission_status - connect to an existing url""" - - authentication_info = get_remote_server_auth(4) - connection = RemoteSwordServer(authentication_info) - result = connection.get_submission_status("") - - self.assertEqual(result != "", False) - - def test_get_submission_status_wrong_url(self): - """Test_get_submission_status - connect to an existing url""" - - authentication_info = get_remote_server_auth(4) - connection = RemoteSwordServer(authentication_info) - result = connection.get_submission_status("http://arxiv.org/reso") - - self.assertEqual(result != "", False) - - -class Test_format_submission_status(InvenioTestCase): - """bibsword_format - test the parsing of format_submission_status method""" - - def test_format_submission_status_submitted(self): - """Test_format_submission_status_submitted""" - - status_xml = open("%s%sTest_submission_status_submitted.xml" % (CFG_TMPDIR, os.sep)).read() - response = format_submission_status(status_xml) - - self.assertEqual(response['status'], "submitted") - self.assertEqual(response['id_submission'], "") - self.assertEqual(response['error'], "") - - - def test_format_submission_status_published(self): - """Test_format_submission_status_published""" - - status_xml = open("%s%sTest_submission_status_published.xml" % (CFG_TMPDIR, os.sep)).read() - response = format_submission_status(status_xml) - - self.assertEqual(response['status'], "published") - self.assertEqual(response['id_submission'], "1003.9876") - self.assertEqual(response['error'], "") - - - def test_format_submission_status_onhold(self): - """Test_format_submission_status_onhold""" - - status_xml = open("%s%sTest_submission_status_onhold.xml" % (CFG_TMPDIR, os.sep)).read() - response = format_submission_status(status_xml) - - self.assertEqual(response['status'], "onhold") - self.assertEqual(response['id_submission'], "") - self.assertEqual(response['error'], "") - - - def test_format_submission_status_removed(self): - """Test_format_submission_status_removed""" - - status_xml = open("%s%sTest_submission_status_unknown.xml" % (CFG_TMPDIR, os.sep)).read() - response = format_submission_status(status_xml) - - self.assertEqual(response['status'], CFG_SUBMISSION_STATUS_REMOVED) - self.assertEqual(response['id_submission'], "") - self.assertEqual(response['error'], "identifier does not correspond to a SWORD wrapper, it may belong to a media deposit") - - -class Test_swrCLIENTDATA_table(InvenioTestCase): - ''' - This test check that the entire update process works fine. It insert some - data into the swrCLIENTDATA table, then he get some xml status entry from - ArXiv and Finally, it update those that have changed their status - ''' - - id_tests = [] - - def test_insert_submission(self): - '''Test_insert_submission - check insert submission rows in swrCLIENTDATA''' - - self.id_tests.append(insert_into_swr_clientdata(1, 97, 'TESLA-FEL-99-07', 10030148, - '6', 'test_username', 'juliet.capulet@cds.cern.ch', - 'test_media_deposit', 'test_media_submit', - 'https://arxiv.org/sword-app/edit/10030148', - 'https://arxiv.org/sword-app/edit/10030148.atom', - 'http://arxiv.org/resolve/app/10030148')) - - self.id_tests.append(insert_into_swr_clientdata(1, 92, 'hep-th/0606096', 10070221, - '5', 'test_username', 'romeo.montague@cds.cern.ch', - 'test_media_deposit', 'test_media_submit', - 'https://arxiv.org/sword-app/edit/10070221', - 'https://arxiv.org/sword-app/edit/10070221.atom', - 'http://arxiv.org/resolve/app/10070221')) - - self.id_tests.append(insert_into_swr_clientdata(1, 92, 'hep-th/0606096', 12340097, - '5', 'test_username', 'romeo.montague@cds.cern.ch', - 'test_media_deposit', 'test_media_submit', - 'https://arxiv.org/sword-app/edit/12340097', - 'https://arxiv.org/sword-app/edit/12340097.atom', - 'http://arxiv.org/resolve/app/12340097')) - - rows = run_sql('''SELECT id, id_swrREMOTESERVER, id_record, report_no, - id_remote, id_user, user_name, user_email, xml_media_deposit, - xml_metadata_submit, submission_date, publication_date, removal_date, - link_medias, link_metadata, link_status, status, last_update - FROM swrCLIENTDATA''') - - for row in rows: - self.assertEqual(row[0] in self.id_tests, True) - if row[0] == self.id_tests[0]: - self.assertEqual(row[1], 1) - self.assertEqual(row[2], 97) - self.assertEqual(row[4], '10030148') - self.assertEqual(row[8], 'test_media_deposit') - self.assertEqual(row[13], "https://arxiv.org/sword-app/edit/10030148") - self.assertEqual(row[14], 'https://arxiv.org/sword-app/edit/10030148.atom') - self.assertEqual(row[15], 'http://arxiv.org/resolve/app/10030148') - self.assertEqual(row[16], CFG_SUBMISSION_STATUS_SUBMITTED) - - if row[0] == self.id_tests[1]: - self.assertEqual(row[1], 1) - self.assertEqual(row[2], 92) - self.assertEqual(row[4], '10070221') - self.assertEqual(row[8], 'test_media_deposit') - self.assertEqual(row[13], 'https://arxiv.org/sword-app/edit/10070221') - self.assertEqual(row[14], 'https://arxiv.org/sword-app/edit/10070221.atom') - self.assertEqual(row[15], 'http://arxiv.org/resolve/app/10070221') - self.assertEqual(row[16], CFG_SUBMISSION_STATUS_SUBMITTED) - - if row[0] == self.id_tests[2]: - self.assertEqual(row[1], 1) - self.assertEqual(row[2], 92) - self.assertEqual(row[4], '12340097') - self.assertEqual(row[8], 'test_media_deposit') - self.assertEqual(row[13], 'https://arxiv.org/sword-app/edit/12340097') - self.assertEqual(row[14], 'https://arxiv.org/sword-app/edit/12340097.atom') - self.assertEqual(row[15], 'http://arxiv.org/resolve/app/12340097') - self.assertEqual(row[16], CFG_SUBMISSION_STATUS_SUBMITTED) - - - def test_update_submission(self): - '''Test_insert_submission - check update submission rows in swrCLIENTDATA''' - - update_submission_status(self.id_tests[0], CFG_SUBMISSION_STATUS_SUBMITTED) - update_submission_status(self.id_tests[1], CFG_SUBMISSION_STATUS_PUBLISHED, '1007.0221') - update_submission_status(self.id_tests[2], CFG_SUBMISSION_STATUS_REMOVED) - - rows = run_sql('''SELECT id, id_swrREMOTESERVER, id_record, report_no, - id_remote, id_user, user_name, user_email, xml_media_deposit, - xml_metadata_submit, submission_date, publication_date, removal_date, - link_medias, link_metadata, link_status, status, last_update - FROM swrCLIENTDATA''') - for row in rows: - self.assertEqual(row[0] in self.id_tests, True) - if row[0] == self.id_tests[0]: - self.assertEqual(row[1], 1) - self.assertEqual(row[2], 97) - self.assertEqual(row[4], '10030148') - self.assertEqual(row[8], 'test_media_deposit') - self.assertEqual(row[13], 'https://arxiv.org/sword-app/edit/10030148') - self.assertEqual(row[14], 'https://arxiv.org/sword-app/edit/10030148.atom') - self.assertEqual(row[15], 'http://arxiv.org/resolve/app/10030148') - self.assertEqual(row[16], CFG_SUBMISSION_STATUS_SUBMITTED) - - if row[0] == self.id_tests[1]: - self.assertEqual(row[1], 1) - self.assertEqual(row[2], 92) - self.assertEqual(row[4], '1007.0221') - self.assertEqual(row[8], 'test_media_deposit') - self.assertEqual(row[13], 'https://arxiv.org/sword-app/edit/10070221') - self.assertEqual(row[14], 'https://arxiv.org/sword-app/edit/10070221.atom') - self.assertEqual(row[15], 'http://arxiv.org/resolve/app/10070221') - self.assertEqual(row[16], CFG_SUBMISSION_STATUS_PUBLISHED) - - if row[0] == self.id_tests[2]: - self.assertEqual(row[1], 1) - self.assertEqual(row[2], 92) - self.assertEqual(row[4], '12340097') - self.assertEqual(row[8], 'test_media_deposit') - self.assertEqual(row[13], 'https://arxiv.org/sword-app/edit/12340097') - self.assertEqual(row[14], 'https://arxiv.org/sword-app/edit/12340097.atom') - self.assertEqual(row[15], 'http://arxiv.org/resolve/app/12340097') - self.assertEqual(row[16], CFG_SUBMISSION_STATUS_REMOVED) - - - def test_yread_submission(self): - '''test_read_submission - check read submission rows in swrCLIENTDATA''' - - - currentDate = time.strftime("%Y-%m-%d %H:%M:%S") - - results = select_submitted_record_infos() - - self.assertEqual(len(results), run_sql('''SELECT COUNT(*) FROM swrCLIENTDATA''')[0][0]) - for result in results: - self.assertEqual(result['id'] in self.id_tests, True) - if result['id'] == self.id_tests[0]: - self.assertEqual(result['id_server'], 1) - self.assertEqual(result['id_record'], 97) - self.assertEqual(result['id_user'], 6) - self.assertEqual(result['id_remote'], '10030148') - self.assertEqual(result['submission_date'], currentDate) - self.assertEqual(result['publication_date'], '') - self.assertEqual(result['removal_date'], "") - self.assertEqual(result['link_medias'], 'https://arxiv.org/sword-app/edit/10030148') - self.assertEqual(result['link_metadata'], 'https://arxiv.org/sword-app/edit/10030148.atom') - self.assertEqual(result['link_status'], 'http://arxiv.org/resolve/app/10030148') - self.assertEqual(result['status'], CFG_SUBMISSION_STATUS_SUBMITTED) - - results = select_submitted_record_infos(4) - - self.assertEqual(len(results), run_sql('''SELECT COUNT(*) FROM swrCLIENTDATA WHERE id_swrREMOTESERVER=4''')[0][0]) - - for result in results: - self.assertEqual(result['id'] in self.id_tests, True) - if result['id'] == self.id_tests[1]: - self.assertEqual(result['id_server'], 4) - self.assertEqual(result['id_record'], 92) - self.assertEqual(result['id_user'], 5) - self.assertEqual(result['id_remote'], '1007.0221') - self.assertEqual(result['submission_date'], currentDate) - self.assertEqual(result['publication_date'], currentDate) - self.assertEqual(result['removal_date'], "") - self.assertEqual(result['link_medias'], 'https://arxiv.org/sword-app/edit/10070221') - self.assertEqual(result['link_metadata'], 'https://arxiv.org/sword-app/edit/10070221.atom') - self.assertEqual(result['link_status'], 'http://arxiv.org/resolve/app/10070221') - self.assertEqual(result['status'], CFG_SUBMISSION_STATUS_PUBLISHED) - - - results = select_submitted_record_infos(row_id=self.id_tests[2]) - self.assertEqual(len(results), 1) - - for result in results: - self.assertEqual(result['id'] in self.id_tests, True) - if result['id'] == self.id_tests[2]: - self.assertEqual(result['id_server'], 1) - self.assertEqual(result['id_record'], 92) - self.assertEqual(result['id_user'], 5) - self.assertEqual(result['id_remote'], '12340097') - self.assertEqual(result['submission_date'], currentDate) - self.assertEqual(result['publication_date'], "") - self.assertEqual(result['removal_date'], currentDate) - self.assertEqual(result['link_medias'], 'https://arxiv.org/sword-app/edit/12340097') - self.assertEqual(result['link_metadata'], 'https://arxiv.org/sword-app/edit/12340097.atom') - self.assertEqual(result['link_status'], 'http://arxiv.org/resolve/app/12340097') - self.assertEqual(result['status'], CFG_SUBMISSION_STATUS_REMOVED) - - - def test_zdelete_submission(self): - '''test_delete_submission - check delete submission rows in swrCLIENTDATA''' - - nb_rows_before = run_sql('''SELECT COUNT(*) FROM swrCLIENTDATA''')[0][0] - - for id_test in self.id_tests: - delete_from_swr_clientdata(id_test) - - nb_rows_after = run_sql('''SELECT COUNT(*) FROM swrCLIENTDATA''')[0][0] - - nb_rows = nb_rows_before - nb_rows_after - - self.assertEqual(nb_rows, 3) - - -class Test_list_submitted_resources(InvenioTestCase): - '''Test_list_submitted_resources - check the data selection and update for admin interface''' - - def test_list_submitted_resources(self): - '''Test_list_submitted_resources - check the data selection and update for admin interface''' - - id_tests = [] - - id_tests.append(insert_into_swr_clientdata(4, 97, 1, 'test', 'test', '10030148', - 'https://arxiv.org/sword-app/edit/10030148', - 'https://arxiv.org/sword-app/edit/10030148.atom', - 'http://arxiv.org/resolve/app/10030148')) - - time.sleep(1) - - id_tests.append(insert_into_swr_clientdata(4, 97, 1, 'test', 'test', '10030148', - 'https://arxiv.org/sword-app/edit/10030148', - 'https://arxiv.org/sword-app/edit/10030148.atom', - 'http://arxiv.org/resolve/app/10030148')) - - update_submission_status(id_tests[1], CFG_SUBMISSION_STATUS_PUBLISHED, '1003.0148') - - time.sleep(1) - - id_tests.append(insert_into_swr_clientdata(3, 92, 2, 'test', 'test', '12340097', - 'https://arxiv.org/sword-app/edit/12340097', - 'https://arxiv.org/sword-app/edit/12340097.atom', - 'http://arxiv.org/resolve/app/12340097')) - - time.sleep(1) - - (submissions, modifications) = list_submitted_resources(0, 10, '') - - self.assertEqual(len(submissions), 3) - self.assertEqual(len(modifications), 2) - - self.assertEqual(submissions[1]['id_remote'], '1003.3743') - self.assertEqual(submissions[1]['status'], CFG_SUBMISSION_STATUS_PUBLISHED) - self.assertEqual(submissions[1]['publication_date'] != '', True) - - self.assertEqual(submissions[2]['id_remote'], '1003.0148') - self.assertEqual(submissions[2]['status'], CFG_SUBMISSION_STATUS_PUBLISHED) - self.assertEqual(submissions[2]['publication_date'] != '', True) - - self.assertEqual(submissions[0]['id_remote'], '12340097') - self.assertEqual(submissions[0]['status'], CFG_SUBMISSION_STATUS_REMOVED) - self.assertEqual(submissions[0]['removal_date'] != '', True) - - self.assertEqual(modifications[1], submissions[0]['id']) - self.assertEqual(modifications[0], submissions[1]['id']) - - - for id_test in id_tests: - delete_from_swr_clientdata(id_test) - - -class Test_format_metadata_atom(InvenioTestCase): - ''' Test_format_metadata_atom - check the generation of the atom entry containing metadata''' - - def test_format_full_metadata(self): - ''' test_format_full_metadata check that every metadata get its xml node''' - - contributors = [] - contributor = {'name' : 'contributor_1_test', - 'email' : 'contributor_1@test.com', - 'affiliation': 'affiliation_test'} - contributors.append(contributor) - - contributor = {'name' : 'contributor_2_test', - 'email' : 'contributor_2@test.com', - 'affiliation': ''} - contributors.append(contributor) - - contributor = {'name' : 'contributor_3_test', - 'email' : '', - 'affiliation' : ''} - contributors.append(contributor) - - categories = [] - category = {'url' : 'https://arxiv.org/test-categories-1', - 'label': 'category_1_test'} - categories.append(category) - category = {'url' : 'https://arxiv.org/test-categories-2', - 'label': 'category_2_test'} - categories.append(category) - category = {'url' : 'https://arxiv.org/test-categories-3', - 'label': 'category_3_test'} - categories.append(category) - - journal_refs = [] - journal_refs.append('journal_ref_test_1') - journal_refs.append('journal_ref_test_2') - - report_nos = [] - report_nos.append('report_no_test_1') - report_nos.append('report_no_test_2') - - links = [] - links.append({'link': 'http://arxiv.org/test_1', - 'type': 'application/test_1'}) - links.append({'link': 'http://arxiv.org/test_2', - 'type': 'application/test_2'}) - - metadata = {'id' : 'id_test', - 'title' : 'title_test', - 'author_name' : 'author_test', - 'author_email': 'author_email', - 'contributors': contributors, - 'summary' : 'summary_test', - 'categories' : categories, - 'primary_url' : 'https://arxiv.org/primary-test-categories', - 'primary_label': 'primary_category_label_test', - 'comment' : '23 pages, 3 chapters 1 test', - 'journal_refs': journal_refs, - 'report_nos' : report_nos, - 'doi' : '10.2349.test.209', - 'links' : links - } - - arXivFormat = ArXivFormat() - metadata_atom = arXivFormat.format_metadata(metadata) - - entry_node = minidom.parseString(metadata_atom) - - - self.assertEqual(metadata_atom != '', True) - - -class Test_submission_process(InvenioTestCase): - '''Test_submission_process - test document submission''' - - def test_perform_submission_process(self): - '''Test_perform_submission_process - test document submission''' - - metadata = {} - metadata['primary_label'] = 'Test - Test Disruptive Networks' - metadata['primary_url'] = 'http://arxiv.org/terms/arXiv/test.dis-nn' - - user_info = {} - user_info['nickname'] = 'test_user' - user_info['email'] = 'test@user.com' - user_info['id'] = 1 - - result = perform_submission_process(4, 'https://arxiv.org/sword-app/test-collection', - 97, user_info, metadata) - - self.assertEqual(open('/tmp/media.xml', 'r').read() != '', True) - self.assertEqual(open('/tmp/metadata.xml', 'r').read() != '', True) - self.assertEqual(open('/tmp/submit.xml', 'r').read() != '', True) - - - - if result['row_id'] != '': - delete_from_swr_clientdata(result['row_id']) - - - -TEST_SUITE = make_test_suite(Test_format_marcxml_file, - Test_format_metadata, - #Test_get_submission_status, - #Test_format_submission_status, - Test_swrCLIENTDATA_table) - #Test_format_metadata_atom) - #Test_list_submitted_resources) - #Test_submission_process) - -if __name__ == "__main__": - run_test_suite(TEST_SUITE) - diff --git a/modules/bibsword/lib/bibsword_config.py b/modules/bibsword/lib/bibsword_config.py index 4df52e48f4..1ce97a6373 100644 --- a/modules/bibsword/lib/bibsword_config.py +++ b/modules/bibsword/lib/bibsword_config.py @@ -1,5 +1,7 @@ +# -*- coding: utf-8 -*- +# # This file is part of Invenio. -# Copyright (C) 2010, 2011, 2013 CERN. +# Copyright (C) 2010, 2011, 2012, 2013. 2014, 2015, 2016 CERN. # # Invenio is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -15,126 +17,76 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. -''' -Forward to ArXiv.org source code -''' +""" +BibSword general configuration variables. +""" from invenio.bibformat_dblayer import get_tag_from_name - -#Maximal time to keep the stored XML Service doucment before reloading it in sec -CFG_BIBSWORD_SERVICEDOCUMENT_UPDATE_TIME = 3600 - -#Default submission status -CFG_SUBMISSION_STATUS_SUBMITTED = "submitted" -CFG_SUBMISSION_STATUS_PUBLISHED = "published" -CFG_SUBMISSION_STATUS_ONHOLD = "onhold" -CFG_SUBMISSION_STATUS_REMOVED = "removed" - -CFG_SUBMIT_ARXIV_INFO_MESSAGE = "Submitted from Invenio to arXiv by %s, on %s, as %s" -CFG_DOCTYPE_UPLOAD_COLLECTION = 'PUSHED_TO_ARXIV' - - -# report number: -marc_tag_main_report_number = get_tag_from_name('primary report number') -if marc_tag_main_report_number: - CFG_MARC_REPORT_NUMBER = marc_tag_main_report_number -else: - CFG_MARC_REPORT_NUMBER = '037__a' - -# title: -marc_tag_title = get_tag_from_name('title') -if marc_tag_title: - CFG_MARC_TITLE = marc_tag_title -else: - CFG_MARC_TITLE = '245__a' - -# author name: -marc_tag_author = get_tag_from_name('first author name') -if marc_tag_author: - CFG_MARC_AUTHOR_NAME = marc_tag_author -else: - CFG_MARC_AUTHOR_NAME = '100__a' - -# author affiliation -marc_tag_author_affiliation = get_tag_from_name('first author affiliation') -if marc_tag_author_affiliation: - CFG_MARC_AUTHOR_AFFILIATION = marc_tag_author_affiliation -else: - CFG_MARC_AUTHOR_AFFILIATION = '100__u' - -# contributor name: -marc_tag_contributor_name = get_tag_from_name('additional author name') -if marc_tag_contributor_name: - CFG_MARC_CONTRIBUTOR_NAME = marc_tag_contributor_name -else: - CFG_MARC_CONTRIBUTOR_NAME = '700__a' - -# contributor affiliation: -marc_tag_contributor_affiliation = get_tag_from_name('additional author affiliation') -if marc_tag_contributor_affiliation: - CFG_MARC_CONTRIBUTOR_AFFILIATION = marc_tag_contributor_affiliation -else: - CFG_MARC_CONTRIBUTOR_AFFILIATION = '700__u' - -# abstract: -marc_tag_abstract = get_tag_from_name('main abstract') -if marc_tag_abstract: - CFG_MARC_ABSTRACT = marc_tag_abstract -else: - CFG_MARC_ABSTRACT = '520__a' - -# additional report number -marc_tag_additional_report_number = get_tag_from_name('additional report number') -if marc_tag_additional_report_number: - CFG_MARC_ADDITIONAL_REPORT_NUMBER = marc_tag_additional_report_number -else: - CFG_MARC_ADDITIONAL_REPORT_NUMBER = '088__a' - -# doi -marc_tag_doi = get_tag_from_name('doi') -if marc_tag_doi: - CFG_MARC_DOI = marc_tag_doi -else: - CFG_MARC_DOI = '909C4a' - -# journal code -marc_tag_journal_ref_code = get_tag_from_name('journal code') -if marc_tag_journal_ref_code: - CFG_MARC_JOURNAL_REF_CODE = marc_tag_journal_ref_code -else: - CFG_MARC_JOURNAL_REF_CODE = '909C4c' - -# journal reference title -marc_tag_journal_ref_title = get_tag_from_name('journal title') -if marc_tag_journal_ref_title: - CFG_MARC_JOURNAL_REF_TITLE = marc_tag_journal_ref_title -else: - CFG_MARC_JOURNAL_REF_TITLE = '909C4p' - -# journal reference page -marc_tag_journal_ref_page = get_tag_from_name('journal page') -if marc_tag_journal_ref_page: - CFG_MARC_JOURNAL_REF_PAGE = marc_tag_journal_ref_page -else: - CFG_MARC_JOURNAL_REF_PAGE = '909C4v' - -# journal reference year -marc_tag_journal_ref_year = get_tag_from_name('journal year') -if marc_tag_journal_ref_year: - CFG_MARC_JOURNAL_REF_YEAR = marc_tag_journal_ref_year -else: - CFG_MARC_JOURNAL_REF_YEAR = '909C4y' - -# comment -marc_tag_comment = get_tag_from_name('comment') -if marc_tag_comment: - CFG_MARC_COMMENT = marc_tag_comment -else: - CFG_MARC_COMMENT = '500__a' - -# internal note field -marc_tag_internal_note = get_tag_from_name('internal notes') -if marc_tag_internal_note: - CFG_MARC_RECORD_SUBMIT_INFO = marc_tag_internal_note -else: - CFG_MARC_RECORD_SUBMIT_INFO = '595__a' +from invenio.urlutils import make_user_agent_string + +# Define the maximum number of contributors to be displayed and submitted +CFG_BIBSWORD_MAXIMUM_NUMBER_OF_CONTRIBUTORS = 30 + +# Define a custom timeout for urllib2 requests +CFG_BIBSWORD_FORCE_DEFAULT_TIMEOUT = True +CFG_BIBSWORD_DEFAULT_TIMEOUT = 20.0 + +# The updated date format according to: +# * http://tools.ietf.org/html/rfc4287#section-3.3 +# * https://dev.mysql.com/doc/refman/5.5/en/date-and-time-functions.html#function_date-format +# NOTE: Specify the local timezone in the "+xx:yy" format. +# If unknown, must be "Z". +CFG_BIBSWORD_UPDATED_DATE_MYSQL_FORMAT = "%Y-%m-%dT%H:%i:%S" +CFG_BIBSWORD_LOCAL_TIMEZONE = "+01:00" + +# The default atom entry mime-type for POST requests +CFG_BIBSWORD_ATOM_ENTRY_MIME_TYPE = "application/atom+xml;type=entry" + +# Must be the same as in the client_servers/Makefile.am +CFG_BIBSWORD_CLIENT_SERVERS_PATH = "bibsword_client_servers" + +CFG_BIBSWORD_USER_AGENT = make_user_agent_string("BibSword") + +CFG_BIBSWORD_MARC_FIELDS = { + 'rn': get_tag_from_name('primary report number') or '037__a', + 'title': get_tag_from_name('title') or '245__a', + 'author_name': get_tag_from_name('first author name') or '100__a', + 'author_affiliation': + get_tag_from_name('first author affiliation') or '100__u', + 'author_email': get_tag_from_name('first author email') or '859__f', + 'contributor_name': + get_tag_from_name('additional author name') or '700__a', + 'contributor_affiliation': + get_tag_from_name('additional author affiliation') or '700__u', + # 'abstract': get_tag_from_name('main abstract') or '520__a', + 'abstract': '520__a', + 'additional_rn': get_tag_from_name('additional report number') or '088__a', + # 'doi': get_tag_from_name('doi') or '909C4a', + 'doi': '909C4a', + # 'journal_code': get_tag_from_name('journal code') or '909C4c', + 'journal_code': '909C4v', + # 'journal_title': get_tag_from_name('journal title') or '909C4p', + 'journal_title': '909C4p', + # 'journal_page': get_tag_from_name('journal page') or '909C4v', + 'journal_page': '909C4c', + # 'journal_year': get_tag_from_name('journal year') or '909C4y', + 'journal_year': '909C4y', + 'comments': get_tag_from_name('comment') or '500__a', + 'internal_notes': get_tag_from_name('internal notes') or '595__a', +} + +CFG_BIBSWORD_FILE_TYPES_MAPPING = { + 'application/atom+xml;type=entry': ['.xml'], + 'application/zip': ['.zip'], + 'application/xml': ['.wsdl', '.xpdl', '.rdf', '.xsl', '.xml', '.xsd'], + 'application/pdf': ['.pdf'], + 'application/postscript': + ['.ai', '.ps', '.eps', '.epsi', '.epsf', '.eps2', '.eps3'], + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': + ['.docx'], + 'text/xml': ['.xml'], + 'image/jpeg': ['.jpe', '.jpg', '.jpeg'], + 'image/jpg': ['.jpg'], + 'image/png': ['.png'], + 'image/gif': ['.gif'], +} diff --git a/modules/bibsword/lib/bibsword_webinterface.py b/modules/bibsword/lib/bibsword_webinterface.py index 8bad0a4245..cc5202d3a8 100644 --- a/modules/bibsword/lib/bibsword_webinterface.py +++ b/modules/bibsword/lib/bibsword_webinterface.py @@ -1,8 +1,7 @@ -''' -Forward to ArXiv.org source code -''' +# -*- coding: utf-8 -*- +# # This file is part of Invenio. -# Copyright (C) 2010, 2011 CERN. +# Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016 CERN. # # Invenio is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -18,496 +17,494 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. -__revision__ = "$Id$" - -__lastupdated__ = """$Date$""" - -import os -from invenio.access_control_engine import acc_authorize_action -from invenio.config import CFG_SITE_URL, CFG_TMPDIR -from invenio.webuser import page_not_authorized, collect_user_info -from invenio.bibsword_client import perform_display_sub_status, \ - perform_display_server_list, \ - perform_display_collection_list, \ - perform_display_category_list, \ - perform_display_metadata, \ - perform_submit_record, \ - perform_display_server_infos, \ - list_remote_servers -from invenio.webpage import page -from invenio.messages import gettext_set_language -from invenio.webinterface_handler import wash_urlargd, WebInterfaceDirectory -from invenio.websubmit_functions.Get_Recid import \ - get_existing_records_for_reportnumber -from invenio.search_engine_utils import get_fieldvalues -from invenio.bibsword_config import CFG_MARC_REPORT_NUMBER, CFG_MARC_ADDITIONAL_REPORT_NUMBER - -class WebInterfaceSword(WebInterfaceDirectory): - """ Handle /bibsword set of pages.""" - _exports = ['', 'remoteserverinfos'] - - - def __init__(self, reqid=None): - '''Initialize''' - self.reqid = reqid - +""" +BibSword Web Interface. +""" + +from invenio.access_control_engine import( + acc_authorize_action +) +import invenio.bibsword_client as sword_client +from invenio.config import( + CFG_SITE_LANG, + CFG_SITE_URL +) +from invenio.messages import( + gettext_set_language +) +from invenio.webinterface_handler import( + wash_urlargd, + WebInterfaceDirectory +) +from invenio.webpage import( + page +) +from invenio.webuser import( + getUid, + page_not_authorized +) - def __call__(self, req, form): - errors = [] - warnings = [] - body = '' - error_messages = [] +__lastupdated__ = """$Date$""" - #*********************************************************************** - # Get values from the form - #*********************************************************************** +class WebInterfaceSwordClient(WebInterfaceDirectory): + """Web interface for the BibSword client.""" + + _exports = [ + "", + "servers", + "server_options", + "submissions", + "submission_options", + "submit", + "submit_step_1", + "submit_step_2", + "submit_step_3", + "submit_step_4", + ] + + def submissions(self, req, form): + """Web interface for the existing submissions.""" + # Check if the user has rights to manage the Sword client + auth_code, auth_message = acc_authorize_action( + req, + "run_sword_client" + ) + if auth_code != 0: + return page_not_authorized( + req=req, + referer="/sword_client/", + text=auth_message, + navtrail="" + ) argd = wash_urlargd(form, { - 'ln' : (str, ''), - - # information of the state of the form submission - 'status' : (str, ''), - 'submit' : (str, ''), - 'last_row' : (str, ''), - 'first_row' : (str, ''), - 'offset' : (int, ''), - 'total_rows' : (str, ''), - - # mendatory informations - 'id_record' : (str, ''), - 'recid' : (int, 0), - 'id_remote_server' : (str, ''), - 'id_collection' : (str, ''), - 'id_primary' : (str, ''), - 'id_categories' : (list, []), - - 'id' : (str, ''), - 'title' : (str, ''), - 'summary' : (str, ''), - 'author_name' : (str, ''), - 'author_email' : (str, ''), - 'contributor_name' : (list, []), - 'contributor_email' : (list, []), - 'contributor_affiliation' : (list, []), - - # optionnal informations - 'comment' : (str, ''), - 'doi' : (str, ''), - 'type' : (str, ''), - 'journal_refs' : (list, []), - 'report_nos' : (list, []), - 'media' : (list, []), - 'new_media' : (str, ''), - 'filename' : (str, '') + "ln": (str, CFG_SITE_LANG), }) - # set language for i18n text auto generation - _ = gettext_set_language(argd['ln']) + # Get the user ID + uid = getUid(req) + + # Set language for i18n text auto generation + ln = argd["ln"] + _ = gettext_set_language(ln) + + body = sword_client.perform_request_submissions( + ln + ) + + navtrail = """ +> %(label)s +""" % { + 'CFG_SITE_URL': CFG_SITE_URL, + 'label': _("Sword Client"), + } + + return page( + title=_("Submissions"), + body=body, + navtrail=navtrail, + lastupdated=__lastupdated__, + req=req, + language=ln + ) + + def submission_options(self, req, form): + """Web interface for the options on the submissions.""" + # Check if the user has rights to manage the Sword client + auth_code, auth_message = acc_authorize_action( + req, + "run_sword_client" + ) + if auth_code != 0: + return page_not_authorized( + req=req, + referer="/sword_client", + text=auth_message, + navtrail="" + ) + argd = wash_urlargd(form, { + "option": (str, ""), + "action": (str, "submit"), + "server_id": (int, 0), + "status_url": (str, ""), + "ln": (str, CFG_SITE_LANG), + }) - #authentication - (auth_code, auth_message) = self.check_credential(req) + if argd["option"] in ("update",): + option = argd["option"] + else: + option = "" + + if argd["action"] in ("submit",): + action = argd["action"] + else: + action = "" + + server_id = argd["server_id"] + status_url = argd["status_url"] + ln = argd["ln"] + + (error, result) = sword_client.perform_request_submission_options( + option, + action, + server_id, + status_url, + ln + ) + + if error: + req.set_content_type("text/plain; charset=utf-8") + req.set_status("400") + req.send_http_header() + req.write("Error: {0}".format(error)) + return + + return result + + def servers(self, req, form): + """Web interface for the available servers.""" + # Check if the user has rights to manage the Sword client + auth_code, auth_message = acc_authorize_action( + req, + "manage_sword_client" + ) if auth_code != 0: - return page_not_authorized(req=req, referer='/bibsword', - text=auth_message, navtrail='') - - - user_info = collect_user_info(req) - - #Build contributor tuples {name, email and affiliation(s)} - contributors = [] - contributor_id = 0 - affiliation_id = 0 - for name in argd['contributor_name']: - contributor = {} - contributor['name'] = name - contributor['email'] = argd['contributor_email'][contributor_id] - contributor['affiliation'] = [] - is_last_affiliation = False - while is_last_affiliation == False and \ - affiliation_id < len(argd['contributor_affiliation']): - if argd['contributor_affiliation'][affiliation_id] == 'next': - is_last_affiliation = True - elif argd['contributor_affiliation'][affiliation_id] != '': - contributor['affiliation'].append(\ - argd['contributor_affiliation'][affiliation_id]) - affiliation_id += 1 - contributors.append(contributor) - contributor_id += 1 - argd['contributors'] = contributors - - - # get the uploaded file(s) (if there is one) - for key, formfields in form.items(): - if key == "new_media" and hasattr(formfields, "filename") and formfields.filename: - filename = formfields.filename - fp = open(os.path.join(CFG_TMPDIR, filename), "w") - fp.write(formfields.file.read()) - fp.close() - argd['media'].append(os.path.join(CFG_TMPDIR, filename)) - argd['filename'] = os.path.join(CFG_TMPDIR, filename) - - # Prepare navtrail - navtrail = '''Admin Area''' \ - % {'CFG_SITE_URL': CFG_SITE_URL} - - title = _("BibSword Admin Interface") - - #*********************************************************************** - # Display admin main page - #*********************************************************************** - - if argd['status'] == '' and argd['recid'] != '' and argd['id_remote_server'] != '': - remote_servers = list_remote_servers(argd['id_remote_server']) - if len(remote_servers) == 0: - error_messages.append("No corresponding remote server could be found") - (body, errors, warnings) = perform_display_server_list( - error_messages, - argd['id_record']) - else: - title = _("Export with BibSword: Step 2/4") - navtrail += ''' > ''' \ - '''SWORD Interface''' % \ - {'CFG_SITE_URL' : CFG_SITE_URL} - (body, errors, warnings) = perform_display_collection_list( - argd['id_remote_server'], - argd['id_record'], - argd['recid'], - error_messages) - - elif argd['status'] == '' or argd['submit'] == "Cancel": - (body, errors, warnings) = perform_display_sub_status() - - elif argd['status'] == 'display_submission': - - if argd['submit'] == 'Refresh all': - (body, errors, warnings) = \ - perform_display_sub_status(1, argd['offset'], "refresh_all") - - elif argd['submit'] == 'Select': - first_row = 1 - (body, errors, warnings) = \ - perform_display_sub_status(first_row, argd['offset']) - - elif argd['submit'] == 'Next': - first_row = int(argd['last_row']) + 1 - (body, errors, warnings) = \ - perform_display_sub_status(first_row, argd['offset']) - - elif argd['submit'] == 'Prev': - first_row = int(argd['first_row']) - int(argd['offset']) - (body, errors, warnings) = \ - perform_display_sub_status(first_row, argd['offset']) - - elif argd['submit'] == 'First': - (body, errors, warnings) = \ - perform_display_sub_status(1, argd['offset']) - - elif argd['submit'] == 'Last': - first_row = int(argd['total_rows']) - int(argd['offset']) + 1 - (body, errors, warnings) = \ - perform_display_sub_status(first_row, argd['offset']) - - - #*********************************************************************** - # Select remote server - #*********************************************************************** - - # when the user validated the metadata, display - elif argd['submit'] == 'New submission': - title = _("Export with BibSword: Step 1/4") - navtrail += ''' > ''' \ - '''SWORD Interface''' % \ - {'CFG_SITE_URL' : CFG_SITE_URL} - - (body, errors, warnings) = \ - perform_display_server_list(error_messages) - - # check if the user has selected a remote server - elif argd['status'] == 'select_server': - title = _("Export with BibSword: Step 1/4") - navtrail += ''' > ''' \ - '''SWORD Interface''' % \ - {'CFG_SITE_URL' : CFG_SITE_URL} - - # check if given id_record exist and convert it in recid - if argd['recid'] != 0: - report_numbers = get_fieldvalues(argd['recid'], CFG_MARC_REPORT_NUMBER) - report_numbers.extend(get_fieldvalues(argd['recid'], CFG_MARC_ADDITIONAL_REPORT_NUMBER)) - if report_numbers: - argd['id_record'] = report_numbers[0] - - elif argd['id_record'] == '': - error_messages.append("You must specify a report number") - - else: - recids = \ - get_existing_records_for_reportnumber(argd['id_record']) - if len(recids) == 0: - error_messages.append(\ - "No document found with the given report number") - elif len(recids) > 1: - error_messages.append(\ - "Several documents have been found with given the report number") - else: - argd['recid'] = recids[0] - - if argd['id_remote_server'] in ['0', '']: - error_messages.append("No remote server was selected") - - if not argd['id_remote_server'] in ['0', '']: - # get the server's name and host - remote_servers = list_remote_servers(argd['id_remote_server']) - if len(remote_servers) == 0: - error_messages.append("No corresponding remote server could be found") - argd['id_remote_server'] = '0' - - if argd['id_remote_server'] in ['0', ''] or argd['recid'] == 0: - (body, errors, warnings) = perform_display_server_list( - error_messages, - argd['id_record']) - - else: - title = _("Export with BibSword: Step 2/4") - (body, errors, warnings) = perform_display_collection_list( - argd['id_remote_server'], - argd['id_record'], - argd['recid'], - error_messages) - - - #*********************************************************************** - # Select collection - #*********************************************************************** - - # check if the user wants to change the remote server - elif argd['submit'] == 'Modify server': - title = _("Export with BibSword: Step 1/4") - navtrail += ''' > ''' \ - '''SWORD Interface''' % \ - {'CFG_SITE_URL' : CFG_SITE_URL} - (body, errors, warnings) = \ - perform_display_server_list(error_messages, argd['id_record']) - - # check if the user has selected a collection - elif argd['status'] == 'select_collection': - title = _("Export with BibSword: Step 2/4") - navtrail += ''' > ''' \ - '''SWORD Interface''' % \ - {'CFG_SITE_URL': CFG_SITE_URL} - if argd['id_collection'] == '0': - error_messages.append("No collection was selected") - (body, errors, warnings) = perform_display_collection_list( - argd['id_remote_server'], - argd['id_record'], - argd['recid'], - error_messages) - - else: - title = _("Export with BibSword: Step 3/4") - (body, errors, warnings) = perform_display_category_list( - argd['id_remote_server'], - argd['id_collection'], - argd['id_record'], - argd['recid'], - error_messages) - - - #*********************************************************************** - # Select primary - #*********************************************************************** - - # check if the user wants to change the collection - elif argd['submit'] == 'Modify collection': - title = _("Export with BibSword: Step 2/4") - navtrail += ''' > ''' \ - '''SWORD Interface''' % \ - {'CFG_SITE_URL': CFG_SITE_URL} - (body, errors, warnings) = perform_display_collection_list( - argd['id_remote_server'], - argd['id_record'], - argd['recid'], - error_messages) - - # check if the user has selected a primary category - elif argd['status'] == 'select_primary_category': - title = _("Export with BibSword: Step 3/4") - navtrail += ''' > ''' \ - '''SWORD Interface''' % \ - {'CFG_SITE_URL' : CFG_SITE_URL} - if argd['id_primary'] == '0': - error_messages.append("No primary category selected") - (body, errors, warnings) = perform_display_category_list( - argd['id_remote_server'], - argd['id_collection'], - argd['id_record'], - argd['recid'], - error_messages) - - else: - title = _("Export with BibSword: Step 4/4") - (body, errors, warnings) = perform_display_metadata(user_info, - str(argd['id_remote_server']), - str(argd['id_collection']), - str(argd['id_primary']), - argd['id_categories'], - argd['id_record'], - argd['recid'], - error_messages) - - #*********************************************************************** - # Check record media and metadata - #*********************************************************************** - - # check if the user wants to change the collection - elif argd['submit'] == 'Modify destination': - title = _("Export with BibSword: Step 3/4") - navtrail += ''' > ''' \ - '''SWORD Interface''' % \ - {'CFG_SITE_URL' : CFG_SITE_URL} - (body, errors, warnings) = perform_display_category_list( - argd['id_remote_server'], - argd['id_collection'], - argd['id_record'], - argd['recid'], - error_messages) - - - # check if the metadata are complet and well-formed - elif argd['status'] == 'check_submission': - title = _("Export with BibSword: Step 4/4") - navtrail += ''' > ''' \ - '''SWORD Interface''' % \ - {'CFG_SITE_URL' : CFG_SITE_URL} - - if argd['submit'] == "Upload": - error_messages.append("Media loaded") - - if argd['id'] == '': - error_messages.append("Id is missing") - - if argd['title'] == '': - error_messages.append("Title is missing") - - if argd['summary'] == '': - error_messages.append("summary is missing") - elif len(argd['summary']) < 25: - error_messages.append("summary must have at least 25 character") - - if argd['author_name'] == '': - error_messages.append("No submitter name specified") - - if argd['author_email'] == '': - error_messages.append("No submitter email specified") - - if len(argd['contributors']) == 0: - error_messages.append("No author specified") - - if len(error_messages) > 0: - - (body, errors, warnings) = perform_display_metadata(user_info, - str(argd['id_remote_server']), - str(argd['id_collection']), - str(argd['id_primary']), - argd['id_categories'], - argd['id_record'], - argd['recid'], - error_messages, - argd) - - - - else: - - title = _("Export with BibSword: Acknowledgement") - - navtrail += ''' > ''' \ - '''SWORD Interface''' % \ - {'CFG_SITE_URL' : CFG_SITE_URL} - - (body, errors, warnings) = perform_submit_record(user_info, - str(argd['id_remote_server']), - str(argd['id_collection']), - str(argd['id_primary']), - argd['id_categories'], - argd['recid'], - argd) - - - # return of all the updated informations to be display - return page(title = title, - body = body, - navtrail = navtrail, - #uid = uid, - lastupdated = __lastupdated__, - req = req, - language = argd['ln'], - #errors = errors, - warnings = warnings, - navmenuid = "yourmessages") - - - def remoteserverinfos(self, req, form): - ''' - This method handle the /bibsword/remoteserverinfos call - ''' + return page_not_authorized( + req=req, + referer="/sword_client/", + text=auth_message, + navtrail="" + ) argd = wash_urlargd(form, { - 'ln' : (str, ''), - 'id' : (str, '') + "ln": (str, CFG_SITE_LANG), }) - #authentication - (auth_code, auth_message) = self.check_credential(req) + # Get the user ID + uid = getUid(req) + + # Set language for i18n text auto generation + ln = argd["ln"] + _ = gettext_set_language(ln) + + body = sword_client.perform_request_servers( + ln + ) + + navtrail = """ +> %(label)s +""" % { + 'CFG_SITE_URL': CFG_SITE_URL, + 'label': _("Sword Client"), + } + + return page( + title=_("Servers"), + body=body, + navtrail=navtrail, + lastupdated=__lastupdated__, + req=req, + language=ln + ) + + def server_options(self, req, form): + """Web interface for the options on the available servers.""" + # Check if the user has rights to manage the Sword client + auth_code, auth_message = acc_authorize_action( + req, + "manage_sword_client" + ) if auth_code != 0: - return page_not_authorized(req=req, referer='/bibsword', - text=auth_message, navtrail='') + return page_not_authorized( + req=req, + referer="/sword_client", + text=auth_message, + navtrail="" + ) + argd = wash_urlargd(form, { + "option": (str, ""), + "action": (str, "submit"), + "server_id": (int, 0), + "sword_client_server_name": (str, ""), + "sword_client_server_engine": (str, ""), + "sword_client_server_username": (str, ""), + "sword_client_server_password": (str, ""), + "sword_client_server_email": (str, ""), + "sword_client_server_update_frequency": (str, ""), + "ln": (str, CFG_SITE_LANG), + }) - body = perform_display_server_infos(argd['id']) + if argd["option"] in ("add", "update", "modify", "delete"): + option = argd["option"] + else: + option = "" + + if argd["action"] in ("prepare", "submit"): + action = argd["action"] + else: + action = "" + + server_id = argd["server_id"] + + server = ( + argd["sword_client_server_name"], + argd["sword_client_server_engine"], + argd["sword_client_server_username"], + argd["sword_client_server_password"], + argd["sword_client_server_email"], + argd["sword_client_server_update_frequency"], + ) + + ln = argd["ln"] + + (error, result) = sword_client.perform_request_server_options( + option, + action, + server_id, + server, + ln + ) + + if error: + req.set_content_type("text/plain; charset=utf-8") + req.set_status("400") + req.send_http_header() + req.write("Error: {0}".format(error)) + return + + return result + + def submit(self, req, form): + """Submit a record using SWORD.""" + # Check if the user has rights to manage the Sword client + auth_code, auth_message = acc_authorize_action( + req, + "run_sword_client" + ) + if auth_code != 0: + return page_not_authorized( + req=req, + referer="/sword_client", + text=auth_message, + navtrail="" + ) - navtrail = ''' > ''' \ - '''SWORD Interface''' % \ - {'CFG_SITE_URL' : CFG_SITE_URL} + argd = wash_urlargd(form, { + "record_id": (int, 0), + "server_id": (int, 0), + "ln": (str, CFG_SITE_LANG), + }) + # Get the user ID + uid = getUid(req) + + # Set language for i18n text auto generation + ln = argd["ln"] + _ = gettext_set_language(ln) + + record_id = argd["record_id"] + server_id = argd["server_id"] + + body = sword_client.perform_submit( + uid, + record_id, + server_id, + ln + ) + + navtrail = """ +> %(label)s +""" % { + 'CFG_SITE_URL': CFG_SITE_URL, + 'label': _("Sword Client"), + } + + return page( + title=_("Submit"), + body=body, + navtrail=navtrail, + lastupdated=__lastupdated__, + req=req, + language=ln + ) + + def submit_step_1(self, req, form): + """Process step 1 in the submission workflow.""" + # Check if the user has adequate rights to run the bibsword client + # TODO: in a more advanced model, also check if the give user has + # rights to the current submission based on the user id and the + # submission id. It would get even more complicated if we + # introduced people that can approve specific submissions etc. + auth_code, auth_message = acc_authorize_action( + req, + "run_sword_client" + ) + if auth_code != 0: + return page_not_authorized( + req=req, + referer="/sword_client", + text=auth_message, + navtrail="" + ) - # return of all the updated informations to be display - return page(title = 'Remote server infos', - body = body, - navtrail = navtrail, - #uid = uid, - lastupdated = __lastupdated__, - req = req, - language = argd['ln'], - errors = '', - warnings = '', - navmenuid = "yourmessages") + argd = wash_urlargd(form, { + 'sid': (str, ''), + 'server_id': (int, 0), + 'ln': (str, CFG_SITE_LANG), + }) + sid = argd['sid'] + server_id = argd['server_id'] + ln = argd['ln'] + + return sword_client.perform_submit_step_1( + sid, + server_id, + ln + ) + + def submit_step_2(self, req, form): + """Process step 2 in the submission workflow.""" + # Check if the user has adequate rights to run the bibsword client + # TODO: in a more advanced model, also check if the give user has + # rights to the current submission based on the user id and the + # submission id. It would get even more complicated if we + # introduced people that can approve specific submissions etc. + auth_code, auth_message = acc_authorize_action( + req, + "run_sword_client" + ) + if auth_code != 0: + return page_not_authorized( + req=req, + referer="/sword_client", + text=auth_message, + navtrail="" + ) - def check_credential(self, req): - ''' - This method check if the user has the right to get into this - function - ''' + argd = wash_urlargd(form, { + 'sid': (str, ""), + 'collection_url': (str, ""), + 'ln': (str, CFG_SITE_LANG), + }) - auth_code, auth_message = acc_authorize_action(req, 'runbibswordclient') - return (auth_code, auth_message) + sid = argd['sid'] + collection_url = argd['collection_url'] + ln = argd['ln'] + + return sword_client.perform_submit_step_2( + sid, + collection_url, + ln + ) + + def submit_step_3(self, req, form): + """Process step 3 in the submission workflow.""" + # Check if the user has adequate rights to run the bibsword client + # TODO: in a more advanced model, also check if the give user has + # rights to the current submission based on the user id and the + # submission id. It would get even more complicated if we + # introduced people that can approve specific submissions etc. + auth_code, auth_message = acc_authorize_action( + req, + "run_sword_client" + ) + if auth_code != 0: + return page_not_authorized( + req=req, + referer="/sword_client", + text=auth_message, + navtrail="" + ) + argd = wash_urlargd(form, { + 'sid': (str, ""), + 'mandatory_category_url': (str, ""), + 'optional_categories_urls': (list, []), + 'ln': (str, CFG_SITE_LANG), + }) - index = __call__ + sid = argd['sid'] + ln = argd['ln'] + mandatory_category_url = argd['mandatory_category_url'] + optional_categories_urls = argd['optional_categories_urls'] + + return sword_client.perform_submit_step_3( + sid, + mandatory_category_url, + optional_categories_urls, + ln + ) + + def submit_step_4(self, req, form): + """Process step 4 in the submission workflow.""" + # Check if the user has adequate rights to run the bibsword client + # TODO: in a more advanced model, also check if the give user has + # rights to the current submission based on the user id and the + # submission id. It would get even more complicated if we + # introduced people that can approve specific submissions etc. + auth_code, auth_message = acc_authorize_action( + req, + "run_sword_client" + ) + if auth_code != 0: + return page_not_authorized( + req=req, + referer="/sword_client", + text=auth_message, + navtrail="" + ) + argd = wash_urlargd(form, { + "sid": (str, ""), + "rn": (str, ""), + "additional_rn": (list, []), + "title": (str, ""), + "author_fullname": (str, ""), + "author_email": (str, ""), + "author_affiliation": (str, ""), + "abstract": (str, ""), + "contributor_fullname": (list, []), + "contributor_email": (list, []), + "contributor_affiliation": (list, []), + "files": (list, []), + "ln": (str, CFG_SITE_LANG), + }) + sid = argd["sid"] + rn = argd["rn"] + additional_rn = argd["additional_rn"] + title = argd["title"] + author_fullname = argd["author_fullname"] + author_email = argd["author_email"] + author_affiliation = argd["author_affiliation"] + abstract = argd["abstract"] + contributor_fullname = argd["contributor_fullname"] + contributor_email = argd["contributor_email"] + contributor_affiliation = argd["contributor_affiliation"] + files_indexes = argd["files"] + ln = argd["ln"] + + return sword_client.perform_submit_step_4( + sid, + ( + rn, + additional_rn, + title, + author_fullname, + author_email, + author_affiliation, + abstract, + contributor_fullname, + contributor_email, + contributor_affiliation, + files_indexes + ), + ln + ) + + index = submissions diff --git a/modules/bibsword/lib/client_servers/Makefile.am b/modules/bibsword/lib/client_servers/Makefile.am new file mode 100644 index 0000000000..1455109aa5 --- /dev/null +++ b/modules/bibsword/lib/client_servers/Makefile.am @@ -0,0 +1,24 @@ +## This file is part of Invenio. +## Copyright (C) 2010, 2011 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +pylibdir = $(libdir)/python/invenio/bibsword_client_servers + +pylib_DATA = arxiv_org.py __init__.py + +EXTRA_DIST = $(pylib_DATA) + +CLEANFILES = *~ *.tmp *.pyc diff --git a/modules/bibsword/lib/client_servers/__init__.py b/modules/bibsword/lib/client_servers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/bibsword/lib/client_servers/arxiv_org.py b/modules/bibsword/lib/client_servers/arxiv_org.py new file mode 100644 index 0000000000..eaafdd7dfe --- /dev/null +++ b/modules/bibsword/lib/client_servers/arxiv_org.py @@ -0,0 +1,846 @@ +# -*- coding: utf-8 -*- +# +# This file is part of Invenio. +# Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016 CERN. +# +# Invenio is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# Invenio is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Invenio; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +""" +arXiv.org SWORD server. +""" + +from cgi import escape +from lxml import etree + +from invenio.bibsword_client_server import SwordClientServer + +__revision__ = "$Id$" + +_LOCAL_SETTINGS = { + 'realm': 'SWORD at arXiv', + 'uri': 'https://arxiv.org/sword-app/', + 'service_document_url': 'https://arxiv.org/sword-app/servicedocument', +} + +CFG_ARXIV_ORG_VERBOSE = True +CFG_ARXIV_ORG_DRY_RUN = True + + +class ArxivOrg(SwordClientServer): + """ + The arXiv.org SWORD server. + """ + + def __init__(self, settings): + """ + Adds the arXiv.org-specific settings before running + SwordClientServer's constructor. + """ + + settings.update(_LOCAL_SETTINGS) + SwordClientServer.__init__(self, settings) + + def _parse_service_document( + self, + service_document_raw + ): + """ + Parses the raw service document (XML) into a python dictionary. + """ + + service_document_parsed = {} + + try: + service_document_etree = etree.XML(service_document_raw) + except etree.XMLSyntaxError: + return service_document_parsed + + # Get the version + version = service_document_etree.itertext( + tag="{http://purl.org/net/sword/}version" + ).next().strip() + service_document_parsed["version"] = version + + # Get the maximum upload size + maximum_file_size_in_kilobytes = service_document_etree.itertext( + tag="{http://purl.org/net/sword/}maxUploadSize" + ).next().strip() + maximum_file_size_in_bytes = int(maximum_file_size_in_kilobytes)*1024 + service_document_parsed["maxUploadSize"] = maximum_file_size_in_bytes + + # Get the verbose + verbose = service_document_etree.itertext( + tag="{http://purl.org/net/sword/}verbose" + ).next().strip() + service_document_parsed["verbose"] = verbose + + # Get the noOp + noOp = service_document_etree.itertext( + tag="{http://purl.org/net/sword/}noOp" + ).next().strip() + service_document_parsed["noOp"] = noOp + + # Get the collections + service_document_parsed["collections"] = {} + collections = service_document_etree.iterdescendants( + tag='{http://www.w3.org/2007/app}collection' + ) + for collection in collections: + collection_url = collection.get('href') + service_document_parsed["collections"][collection_url] = {} + + # Get the collection title + collection_title = collection.itertext( + tag='{http://www.w3.org/2005/Atom}title' + ).next().strip() + service_document_parsed[ + "collections" + ][ + collection_url + ][ + "title" + ] = collection_title + + # Get the collection accepted file types + service_document_parsed[ + "collections" + ][ + collection_url + ][ + "accepts" + ] = [] + collection_accepts = collection.iterdescendants( + tag='{http://www.w3.org/2007/app}accept' + ) + for collection_accept in collection_accepts: + collection_accept_extensions = \ + SwordClientServer._get_extension_for_file_type( + collection_accept.text.strip() + ) + service_document_parsed[ + "collections" + ][ + collection_url + ][ + "accepts" + ].extend(collection_accept_extensions) + + service_document_parsed[ + "collections" + ][ + collection_url + ][ + "accepts" + ] = tuple( + set( + service_document_parsed[ + "collections" + ][ + collection_url + ][ + "accepts" + ] + ) + ) + + # Get the collection policy + collection_policy = collection.itertext( + tag="{http://purl.org/net/sword/}collectionPolicy" + ).next().strip() + service_document_parsed[ + "collections" + ][ + collection_url + ][ + "policy" + ] = collection_policy + + # Get the collection abstract + collection_abstract = collection.itertext( + tag="{http://purl.org/dc/terms/}abstract" + ).next().strip() + service_document_parsed[ + "collections" + ][ + collection_url + ][ + "abstract" + ] = collection_abstract + + # Get the collection mediation + collection_mediation = collection.itertext( + tag="{http://purl.org/net/sword/}mediation" + ).next().strip() + service_document_parsed[ + "collections" + ][ + collection_url + ][ + "mediation" + ] = collection_mediation + + # Get the collection treatment + collection_treatment = collection.itertext( + tag="{http://purl.org/net/sword/}treatment" + ).next().strip() + service_document_parsed[ + "collections" + ][ + collection_url + ][ + "treatment" + ] = collection_treatment + + # Get the collection categories + service_document_parsed[ + "collections" + ][ + collection_url + ][ + "categories" + ] = {} + collection_categories = collection.iterdescendants( + tag='{http://www.w3.org/2005/Atom}category' + ) + for collection_category in collection_categories: + collection_category_term = collection_category.get( + 'term' + ).strip() + collection_category_scheme = collection_category.get( + 'scheme' + ).strip() + collection_category_label = collection_category.get( + 'label' + ).strip() + service_document_parsed[ + "collections" + ][ + collection_url + ][ + "categories" + ][ + collection_category_term + ] = { + "label": collection_category_label, + "scheme": collection_category_scheme, + } + + # Get the collection primary categories + service_document_parsed[ + "collections" + ][ + collection_url + ][ + "primary_categories" + ] = {} + collection_primary_categories = collection.iterdescendants( + tag='{http://arxiv.org/schemas/atom/}primary_category' + ) + for collection_primary_category in collection_primary_categories: + collection_primary_category_term = \ + collection_primary_category.get( + 'term' + ).strip() + collection_primary_category_scheme = \ + collection_primary_category.get( + 'scheme' + ).strip() + collection_primary_category_label = \ + collection_primary_category.get( + 'label' + ).strip() + service_document_parsed[ + "collections" + ][ + collection_url + ][ + "primary_categories" + ][ + collection_primary_category_term + ] = { + "label": collection_primary_category_label, + "scheme": collection_primary_category_scheme, + } + + return service_document_parsed + + def get_collections(self): + """ + """ + return self.service_document_parsed.get("collections") + + def get_categories(self, collection_url): + """ + """ + return { + "mandatory": self.service_document_parsed.get( + "collections", + ).get( + collection_url, + {} + ).get( + "primary_categories" + ), + "optional": self.service_document_parsed.get( + "collections" + ).get( + collection_url, + {} + ).get( + "categories" + ), + } + + def get_accepted_file_types(self, collection_url): + """ + """ + return self.service_document_parsed.get( + "collections" + ).get( + collection_url, + {} + ).get( + "accepts" + ) + + def get_maximum_file_size(self): + """ + Return size in bytes. + """ + return self.service_document_parsed.get("maxUploadSize") + + def _prepare_media(self, media): + """ + TODO: this function should decide if and how to consolidate the files + depending on * the maximum file size + and * the accepted packaging options + """ + + return media + + def _prepare_media_headers(self, file_info, metadata): + """ + """ + + headers = SwordClientServer._prepare_media_headers( + self, + file_info, + metadata + ) + + headers['Host'] = 'arxiv.org' + + if file_info['checksum']: + headers['Content-MD5'] = file_info['checksum'] + + # NOTE: It looks like media deposit only accepts the "X-On-Behalf-Of" + # header when both the author_name and author_email are present + # in the given format: + # "A. Scientist" . + (author_name, author_email, dummy) = metadata.get( + 'author', + (None, None, None) + ) + if author_email: + if author_name and self._is_ascii(author_name): + headers['X-On-Behalf-Of'] = "\"%s\" <%s>" % ( + author_name, + author_email + ) + else: + headers['X-On-Behalf-Of'] = "%s" % (author_email,) + + if CFG_ARXIV_ORG_VERBOSE: + headers['X-Verbose'] = 'True' + + if CFG_ARXIV_ORG_DRY_RUN: + headers['X-No-Op'] = 'True' + + return headers + + def _prepare_media_response(self, response): + """ + """ + + # NOTE: Could we get the media_link from the response headers? + # media_link = response.headers.get('Location') + + try: + response_xml = response.read() + response.close() + response_etree = etree.XML(response_xml) + response_links = response_etree.findall( + '{http://www.w3.org/2005/Atom}link' + ) + for response_link in response_links: + if response_link.attrib.get('rel') == 'edit-media': + media_link = response_link.attrib.get('href') + break + except: + media_link = None + + return media_link + + def _prepare_media_response_error(self, error): + """ + """ + + try: + error_xml = error.read() + error.close() + error_etree = etree.XML(error_xml) + error_msg = error_etree.findtext( + '{http://www.w3.org/2005/Atom}summary' + ) + if not error_msg: + raise Exception + return error_msg + except: + return "HTTP Error %s: %s" (error.code, error.msg) + + def _prepare_metadata(self, metadata, media_response): + """ + """ + + # Namespaces + # TODO: de-hardcode the namespaces + atom_ns = 'http://www.w3.org/2005/Atom' + atom_nsmap = {None: atom_ns} + arxiv_ns = 'http://arxiv.org/schemas/atom/' + arxiv_nsmap = {'arxiv': arxiv_ns} + + # The root element of the atom entry + entry_element = etree.Element('entry', nsmap=atom_nsmap) + + # The title element + # (SWORD/APP - arXiv.org mandatory) + # TODO: This is mandatory, we shouldn't check + # at this point if it's there or not. + title = metadata['title'] + if title: + title_element = etree.Element('title') + title_element.text = escape(title, True).decode('utf-8') + entry_element.append(title_element) + + # The id element + # (SWORD/APP mandatory) + # TODO: This is mandatory, we shouldn't check + # at this point if it's there or not. + recid = metadata['recid'] + if recid: + id_element = etree.Element('id') + id_element.text = str(recid) + entry_element.append(id_element) + + # The updated element + # (SWORD/APP mandatory) + # TODO: This is mandatory, we shouldn't check + # at this point if it's there or not. + modification_date = metadata['modification_date'] + if modification_date: + updated_element = etree.Element('updated') + updated_element.text = modification_date + entry_element.append(updated_element) + + # The author element + # (arXiv.org mandatory) + # NOTE: The author element is used for the authenticated user, + # i.e. the person who initiates the submission. + # The repeatable contributor element is used to specify the + # individual authors of the material being deposited to arXiv. + # At least one /contributor/email node must be present in order + # to inform arXiv of the email address of the primary contact + # author. If multiple /contributor/email nodes are found, + # the first will be used. Optionally the primary contact author's + # (name and) email address can also be specified in the + # X-On-Behalf-Of HTTP header extension, e.g.: + # X-On-Behalf-Of: "A. Scientist" + # (http://arxiv.org/help/submit_sword#Ingestion) + author_element = etree.Element('author') + author_name_element = etree.Element('name') + author_name_element.text = escape(self.username, True).decode('utf-8') + author_element.append(author_name_element) + author_email_element = etree.Element('email') + author_email_element.text = escape(self.email, True).decode('utf-8') + author_element.append(author_email_element) + entry_element.append(author_element) + + # The contributors element(s) + # (arXiv.org mandatory) + # TODO: This is mandatory, we shouldn't check + # at this point if it's there or not. + # NOTE: At least one /contributor/email node must be present in order + # to inform arXiv of the email address of the primary contact + # author. + (author_name, author_email, author_affiliation) = metadata['author'] + if author_name or author_email or author_affiliation: + contributor_element = etree.Element('contributor') + if author_name: + contributor_name_element = etree.Element('name') + contributor_name_element.text = escape( + author_name, + True + ).decode('utf-8') + contributor_element.append(contributor_name_element) + if author_email: + contributor_email_element = etree.Element('email') + contributor_email_element.text = escape( + author_email, + True + ).decode('utf-8') + contributor_element.append(contributor_email_element) + # TODO: Remove this hack with something more elegant. + else: + contributor_email_element = etree.Element('email') + contributor_email_element.text = escape( + self.email, + True + ).decode('utf-8') + contributor_element.append(contributor_email_element) + if author_affiliation: + contributor_affiliation_element = etree.Element( + '{%s}affiliation' % (arxiv_ns,), + nsmap=arxiv_nsmap + ) + contributor_affiliation_element.text = escape( + author_affiliation, + True + ).decode('utf-8') + contributor_element.append(contributor_affiliation_element) + entry_element.append(contributor_element) + + contributors = metadata['contributors'] + for (contributor_name, + contributor_email, + contributor_affiliation + ) in contributors: + contributor_element = etree.Element('contributor') + if contributor_name: + contributor_name_element = etree.Element('name') + contributor_name_element.text = escape( + contributor_name, + True + ).decode('utf-8') + contributor_element.append(contributor_name_element) + if contributor_email: + contributor_email_element = etree.Element('email') + contributor_email_element.text = escape( + contributor_email, + True + ).decode('utf-8') + contributor_element.append(contributor_email_element) + if contributor_affiliation: + contributor_affiliation_element = etree.Element( + '{%s}affiliation' % (arxiv_ns,), + nsmap=arxiv_nsmap + ) + contributor_affiliation_element.text = escape( + contributor_affiliation, + True + ).decode('utf-8') + contributor_element.append(contributor_affiliation_element) + entry_element.append(contributor_element) + + # The content element + # (arXiv.org optional) + # NOTE: Contains or links to the complete content of the entry. + # Content must be provided if there is no alternate link, + # and should be provided if there is no summary. + # (http://www.atomenabled.org/developers/syndication/#recommendedEntryElements) + + # The summary element + # (arXiv.org mandatory) + # TODO: This is mandatory, we shouldn't check + # at this point if it's there or not. + # The same goes for the minimum length of 20. + abstract = metadata['abstract'] + if abstract: + # TODO: Replace this hack with something more elegant. + if len(abstract) < 20: + abstract += '.' * (20 - len(abstract)) + summary_element = etree.Element('summary') + summary_element.text = escape(abstract, True).decode('utf-8') + entry_element.append(summary_element) + + # The category element(s) + # (arXiv.org optional) + optional_categories = metadata['optional_categories'] + for optional_category in optional_categories: + optional_category_element = etree.Element( + 'category', + term=optional_category['term'], + scheme=optional_category['scheme'], + label=optional_category['label'], + ) + entry_element.append(optional_category_element) + + # The primary_category element + # (arXiv.org mandatory) + mandatory_category = metadata['mandatory_category'] + mandatory_category_element = etree.Element( + '{%s}primary_category' % (arxiv_ns,), nsmap=arxiv_nsmap, + term=mandatory_category['term'], + scheme=mandatory_category['scheme'], + label=mandatory_category['label'], + ) + entry_element.append(mandatory_category_element) + + # The report_no element(s) + # (arXiv.org optional) + rn = metadata['rn'] + if rn: + report_no_element = etree.Element( + '{%s}report_no' % (arxiv_ns,), + nsmap=arxiv_nsmap + ) + report_no_element.text = escape(rn, True).decode('utf-8') + entry_element.append(report_no_element) + + additional_rn = metadata['additional_rn'] + for rn in additional_rn: + report_no_element = etree.Element( + '{%s}report_no' % (arxiv_ns,), + nsmap=arxiv_nsmap + ) + report_no_element.text = escape(rn, True).decode('utf-8') + entry_element.append(report_no_element) + + # The doi element + # (arXiv.org optional) + doi = metadata['doi'] + if doi: + doi_element = etree.Element( + '{%s}doi' % (arxiv_ns,), + nsmap=arxiv_nsmap + ) + doi_element.text = escape(doi, True).decode('utf-8') + entry_element.append(doi_element) + + # The journal_ref element(s) + # (arXiv.org optional) + ( + journal_code, + journal_title, + journal_page, + journal_year + ) = metadata['journal_info'] + if journal_title: + journal_info = "%s" % (journal_title,) + if journal_code: + journal_info += ": %s" % (journal_code,) + if journal_year: + journal_info += " (%s)" % (journal_year,) + if journal_page: + journal_info += " pp. %s" % (journal_page,) + journal_ref_element = etree.Element( + '{%s}journal_ref' % (arxiv_ns,), + nsmap=arxiv_nsmap + ) + journal_ref_element.text = escape( + journal_info, True + ).decode('utf-8') + entry_element.append(journal_ref_element) + + # The comment element + # (arXiv.org optional) + # NOTE: The element contains the typical author + # comments found on most arXiv articles: + # "23 pages, 8 figures and 4 tables" + # TODO: How does this map to metadata['comments'] + # and metadata['internal_notes']? + + # The link element(s) + # (arXiv.org mandatory) + for file_response in media_response.itervalues(): + if not file_response['error']: + link_element = etree.Element( + 'link', + href=file_response['msg'], + type=file_response['mime'], + rel='related' + ) + entry_element.append(link_element) + + prepared_metadata = etree.tostring( + entry_element, + xml_declaration=True, + encoding='utf-8', + pretty_print=True + ) + + return prepared_metadata + + def _prepare_metadata_headers(self, metadata): + """ + """ + + headers = SwordClientServer._prepare_metadata_headers( + self, + metadata + ) + + headers['Host'] = 'arxiv.org' + + # NOTE: It looks like media deposit only accepts the "X-On-Behalf-Of" + # header when both the author_name and author_email are present + # in this format: "A. Scientist" . + ( + author_name, + author_email, + dummy + ) = metadata.get('author', (None, None, None)) + if author_email: + if author_name and self._is_ascii(author_name): + headers['X-On-Behalf-Of'] = "\"%s\" <%s>" % ( + author_name, + author_email + ) + else: + headers['X-On-Behalf-Of'] = "%s" % (author_email,) + + if CFG_ARXIV_ORG_VERBOSE: + headers['X-Verbose'] = 'True' + + if CFG_ARXIV_ORG_DRY_RUN: + headers['X-No-Op'] = 'True' + + return headers + + def _prepare_metadata_response(self, response): + """ + """ + + links = {} + + try: + response_xml = response.read() + response.close() + response_etree = etree.XML(response_xml) + response_links = response_etree.findall( + '{http://www.w3.org/2005/Atom}link' + ) + for response_link in response_links: + if response_link.attrib.get('rel') == 'alternate': + links['alternate'] = response_link.attrib.get('href') + if response_link.attrib.get('rel') == 'edit': + links['edit'] = response_link.attrib.get('href') + except: + links['alternate'] = None + links['edit'] = None + + return links + + def _prepare_metadata_response_error(self, error): + """ + """ + + try: + error_xml = error.read() + error.close() + error_etree = etree.XML(error_xml) + error_msg = error_etree.findtext( + '{http://www.w3.org/2005/Atom}summary' + ) + if not error_msg: + raise Exception + return error_msg + except: + return "HTTP Error %s: %s" (error.code, error.msg) + + def _prepare_response(self, media_response, metadata_response): + """ + """ + + response = {} + + # Update the general response with the metadata response + response.update(metadata_response) + + # If there were errors with the media, + # also update the general response with the media response + if metadata_response['error'] and media_response: + for file_response in media_response.itervalues(): + if file_response['error']: + response['media_response'] = media_response + break + + return response + + def _prepare_status_response(self, response): + """ + """ + + status_and_error = {} + + try: + response_xml = response.read() + response.close() + response_etree = etree.XML(response_xml) + response_status = response_etree.findall('status') + if response_status: + status_and_error['status'] = response_status[0].text + if status_and_error['status'] == "published": + response_arxiv_id = response_etree.findall('arxiv_id') + if response_arxiv_id: + status_and_error[ + 'status' + ] = "{2}".format( + "http://arxiv.org/abs/", + response_arxiv_id[0].text, + "published" + ) + else: + status_and_error['status'] = None + response_error = response_etree.findall('error') + if response_error: + status_and_error['error'] = response_error[0].text + else: + status_and_error['error'] = None + except: + status_and_error['status'] = None + status_and_error['error'] = None + + return status_and_error + + @staticmethod + def _is_ascii(s): + """ + """ + + try: + s.decode("ascii") + except: + return False + else: + return True + +# NOTE: in order to edit an existing submission: +# * First all the media resources should be individually posted +# or deposited packed together into a zip file. +# * Then a metadata wrapper is PUT to the link with "rel=/edit/" +# which was part of the atom entry response to the +# original wrapper deposit. +# ... +# request = urllib2.Request('http://example.org', data='your_put_data') +# request.get_method = lambda: 'PUT' +# ... + + +def arxiv_org(settings): + """ + Instantiates the remote SWORD server. + """ + + return ArxivOrg(settings) diff --git a/modules/webstyle/css/invenio.css b/modules/webstyle/css/invenio.css index a8af047578..2a601eee84 100644 --- a/modules/webstyle/css/invenio.css +++ b/modules/webstyle/css/invenio.css @@ -4263,6 +4263,7 @@ a.author_orcid_image_link { margin-right:10px; } +<<<<<<< HEAD /* WebComment filtering and related files */ #tools-and-filtering-wrapper{ margin: 0px; @@ -4468,4 +4469,260 @@ ul.record_recommendation li{ margin-top:5px; } +/* BibSword START */ +table.bibsword_table { + border: 1px solid #FFCC00; + border-spacing: 0px; + border-collapse: separate; + box-shadow: 3px 3px 9px #AAAAAA; +} + +table.bibsword_table td, table.bibsword_table th { + padding: 10px; + border: 1px solid #FFCC00; +} + +table.bibsword_table thead { + background: #FFFFCC; +} + +a.bibsword_anchor, a.bibsword_anchor:link, a.bibsword_anchor:visited, a.bibsword_anchor:active { + text-decoration: none; +} + +a.bibsword_anchor:hover { + text-decoration: none; + color: red; +} + +div.bibsword_modal_outer { + position: absolute; + left: 0; + right: 0; + margin-left: auto; + margin-right: auto; + top: 0; + bottom: 0; + margin-top: auto; + margin-bottom: auto; + height: 100%; + width: 100%; + background: rgba(66,66,66,0.66); + z-index: 8888; +} + +div.bibsword_modal_inner { + position: absolute; + overflow: auto; + left: 0; + right: 0; + margin-left: auto; + margin-right: auto; + top: 0; + bottom: 0; + margin-top: auto; + margin-bottom: auto; + width: 35%; + min-width: 500px; + max-width: 600px; + height: 75%; + min-height: 600px; + max-height: 800px; + border: 2px solid #FFCC00; + border-radius: 10px; + background: rgba(246, 246, 246, 1); + -webkit-box-shadow: 5px 5px 10px 0px rgba(0,0,0,0.66); + -moz-box-shadow: 5px 5px 10px 0px rgba(0,0,0,0.66); + box-shadow: 5px 5px 10px 0px rgba(0,0,0,0.66); + z-index: 9999; +} + +div.bibsword_modal_label { + width: 150px; + padding: 10px; + border-bottom: 2px solid #FFCC00; + border-right: 2px solid #FFCC00; + background-color: #FFFFCC; + border-top-left-radius: 9px; + border-bottom-right-radius: 10px; + text-align: center; + font-weight: bold; +} + +div.bibsword_modal_content { + padding: 10px; +} + +div.bibsword_modal_content label.bibsword_modal_content_field_title { + font-size: 100%; + color: black; + font-style: normal; + margin-left: 0px; +} +div.bibsword_modal_content div.bibsword_modal_content_field_subtitle { + font-size: 75%; + color: grey; + font-style: italic; + margin-left: 5px; +} + +div.bibsword_modal_content input, select { + padding: 5px; + margin-left: 5px; + margin-top: 5px; + margin-bottom: 15px; +} + +div.bibsword_modal_controls { + padding: 10px; + text-align: right; +} + +div.bibsword_modal_controls button { + padding: 5px; +} + +div.bibsword_submit_container { + padding: 0px; + margin: 0px; +} + +div.bibsword_submit_header { + padding: 5px; + margin-bottom: 10px; +} + +.bibsword_submit_step_title { + border: 1px #FFCC00 solid; + padding: 5px; + margin-bottom: 10px; + font-weight: bold; + background-color: #FFFFCC; +} + +.bibsword_submit_step_details { + padding: 5px; + margin-bottom: 10px; +} + +.bibsword_submit_step_details label { + font-weight: bold; +} + +.bibsword_submit_step_details span.warning { + font-style: italic; + font-weight: normal; + font-size: small; + padding-left: 10px; + color: red; + margin-bottom: 2px; +} + +.bibsword_submit_step_details div.bibsword_submit_step_details_field { + margin-bottom: 20px; +} + +#bibsword_submit_step_4_additional_rn_sortable div.highlight, #bibsword_submit_step_4_contributors_sortable div.highlight { + border: 1px dotted grey; + height: 30px; + background: lightgrey; +} + +.bibsword_submit_step_details div.bibsword_submit_step_details_label { + margin-bottom: 5px; +} + +.bibsword_submit_step_details_input { + margin-bottom: 3px; +} + +.bibsword_submit_step_details_input label { + font-style: italic; + font-weight: normal; + font-size: small; + padding-left: 10px; + color: grey; + margin-bottom: 2px; +} + +.bibsword_submit_step_details_input span.sortable { + cursor: move; + font-size: large; +} + +.bibsword_submit_step_details_input .mandatory { + border-top: 2px inset red; + border-left: 2px inset red; +} + +.bibsword_submit_step_details_input button.negative { + color: red; +} + +.bibsword_submit_step_details_input button.positive { + color: green; +} + +.bibsword_submit_step_details_legend { +} + +.bibsword_submit_step_details_legend ul { + margin: 0px; + padding: 0px; +} + +.bibsword_submit_step_details_legend_item { + background-color: #FFFFCC; + border: 1px #FFCC00 dashed; + border-radius: 15px; + -moz-border-radius: 15px; + -webkit-border-radius: 15px; + -o-border-radius: 15px; + padding-top: 3px; + padding-bottom: 3px; + padding-left: 5px; + padding-right: 5px; + margin: 3px; + font-size: small; + color: #555555; + display: inline-block; +} + +.bibsword_submit_step_details_legend_item img { + vertical-align: bottom; + cursor: pointer; +} +/* BibSword END */ + +/* Generic START */ +.text_align_center { + text-align: center; +} + +.font_size_150percent { + font-size: 150%; +} + +.font_size_75percent { + font-size: 75%; +} + +.margin_10px { + margin: 10px; +} + +a.temporarily_disabled, button.temporarily_disabled { + color: grey !important; + cursor: progress !important; +} + +div.display_inline { + display: inline; +} + +div.display_inline_block { + display: inline-block; +} +/* Generic END */ + /* end of invenio.css */ diff --git a/modules/webstyle/lib/webinterface_layout.py b/modules/webstyle/lib/webinterface_layout.py index 2430c15e03..f7455fe65f 100644 --- a/modules/webstyle/lib/webinterface_layout.py +++ b/modules/webstyle/lib/webinterface_layout.py @@ -247,11 +247,10 @@ def _lookup(self, component, path): WebInterfaceBatchUploaderPages = WebInterfaceDumbPages try: - from invenio.bibsword_webinterface import \ - WebInterfaceSword + from invenio.bibsword_webinterface import WebInterfaceSwordClient except: register_exception(alert_admin=True, subject='EMERGENCY') - WebInterfaceSword = WebInterfaceDumbPages + WebInterfaceSwordClient = WebInterfaceDumbPages try: from invenio.ping_webinterface import \ @@ -423,7 +422,7 @@ class WebInterfaceInvenio(WebInterfaceSearchInterfacePages): 'exporter', 'kb', 'batchuploader', - 'bibsword', + 'sword_client', 'ping', 'admin2', 'linkbacks', @@ -468,7 +467,7 @@ def __init__(self): kb = WebInterfaceBibKnowledgePages() admin2 = WebInterfaceDisabledPages() batchuploader = WebInterfaceDisabledPages() - bibsword = WebInterfaceDisabledPages() + sword_client = WebInterfaceDisabledPages() ping = WebInterfacePingPages() linkbacks = WebInterfaceDisabledPages() author = WebInterfaceDisabledPages() @@ -510,7 +509,7 @@ def __init__(self): kb = WebInterfaceBibKnowledgePages() admin2 = WebInterfaceAdminPages() batchuploader = WebInterfaceBatchUploaderPages() - bibsword = WebInterfaceSword() + sword_client = WebInterfaceSwordClient() ping = WebInterfacePingPages() linkbacks = WebInterfaceRecentLinkbacksPages() author = WebInterfaceAuthor() From f7a0649a52a10493f169934dfef2780af84b4519 Mon Sep 17 00:00:00 2001 From: Sebastian Witowski Date: Wed, 21 Sep 2016 11:05:10 +0200 Subject: [PATCH 57/59] BibSword: Minor fixes for CDS production * Updates bibsword parameters in bfe_sword_push. * Turns off the dry run and verbose options for arxiv. * Updates the link in 'Administration' menu. --- .../bibformat/lib/elements/bfe_sword_push.py | 8 +- .../bibsword/lib/client_servers/arxiv_org.py | 4 +- .../webaccess/lib/access_control_config.py | 2 +- modules/webstyle/lib/webinterface_layout.py | 191 ++++++++++-------- 4 files changed, 113 insertions(+), 92 deletions(-) diff --git a/modules/bibformat/lib/elements/bfe_sword_push.py b/modules/bibformat/lib/elements/bfe_sword_push.py index fd170eca37..76b3b659c7 100644 --- a/modules/bibformat/lib/elements/bfe_sword_push.py +++ b/modules/bibformat/lib/elements/bfe_sword_push.py @@ -38,14 +38,12 @@ def format_element(bfo, remote_server_id, link_label="Push via Sword"): return "" sword_arguments = {'ln': bfo.lang, - 'recid': bfo.recID} + 'record_id': bfo.recID} if remote_server_id: - sword_arguments['id_remote_server'] = remote_server_id - else: - sword_arguments['status'] = 'select_server' + sword_arguments['server_id'] = remote_server_id - return create_html_link(CFG_BASE_URL + '/bibsword', + return create_html_link(CFG_BASE_URL + '/sword_client/submit', sword_arguments, link_label) diff --git a/modules/bibsword/lib/client_servers/arxiv_org.py b/modules/bibsword/lib/client_servers/arxiv_org.py index eaafdd7dfe..f2896a0a58 100644 --- a/modules/bibsword/lib/client_servers/arxiv_org.py +++ b/modules/bibsword/lib/client_servers/arxiv_org.py @@ -34,8 +34,8 @@ 'service_document_url': 'https://arxiv.org/sword-app/servicedocument', } -CFG_ARXIV_ORG_VERBOSE = True -CFG_ARXIV_ORG_DRY_RUN = True +CFG_ARXIV_ORG_VERBOSE = False +CFG_ARXIV_ORG_DRY_RUN = False class ArxivOrg(SwordClientServer): diff --git a/modules/webaccess/lib/access_control_config.py b/modules/webaccess/lib/access_control_config.py index 6e158c0ec4..86388c621b 100644 --- a/modules/webaccess/lib/access_control_config.py +++ b/modules/webaccess/lib/access_control_config.py @@ -654,7 +654,7 @@ class InvenioWebAccessFireroleError(Exception): 'runbibeditmulti' : (_("Run Multi-Record Editor"), "%s/%s/multiedit/?ln=%%s" % (CFG_SITE_URL, CFG_SITE_RECORD)), 'runbibdocfile' : (_("Run Document File Manager"), "%s/%s/managedocfiles?ln=%%s" % (CFG_SITE_URL, CFG_SITE_RECORD)), 'runbibmerge' : (_("Run Record Merger"), "%s/%s/merge/?ln=%%s" % (CFG_SITE_URL, CFG_SITE_RECORD)), - 'runbibswordclient' : (_("Run BibSword client"), "%s/bibsword/?ln=%%s" % CFG_SITE_URL), + 'runbibswordclient' : (_("Run BibSword client"), "%s/sword_client/submissions?ln=%%s" % CFG_SITE_URL), 'cfgbibknowledge' : (_("Configure BibKnowledge"), "%s/kb?ln=%%s" % CFG_SITE_URL), 'cfgbibformat' : (_("Configure BibFormat"), "%s/admin/bibformat/bibformatadmin.py?ln=%%s" % CFG_SITE_URL), 'cfgoaiharvest' : (_("Configure OAI Harvest"), "%s/admin/oaiharvest/oaiharvestadmin.py?ln=%%s" % CFG_SITE_URL), diff --git a/modules/webstyle/lib/webinterface_layout.py b/modules/webstyle/lib/webinterface_layout.py index f7455fe65f..2ddbf7a928 100644 --- a/modules/webstyle/lib/webinterface_layout.py +++ b/modules/webstyle/lib/webinterface_layout.py @@ -1,21 +1,21 @@ # -*- coding: utf-8 -*- -# This file is part of Invenio. -# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012 CERN. -# -# Invenio is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of the -# License, or (at your option) any later version. -# -# Invenio is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Invenio; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. +## This file is part of Invenio. +## Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """ Global organisation of the application's URLs. @@ -28,10 +28,7 @@ from invenio.errorlib import register_exception from invenio.webinterface_handler import WebInterfaceDirectory from invenio import webinterface_handler_config as apache -from invenio.config import CFG_DEVEL_SITE, \ - CFG_OPENAIRE_SITE, \ - CFG_ACCESS_CONTROL_LEVEL_SITE, \ - CFG_CERN_SITE +from invenio.config import CFG_DEVEL_SITE, CFG_OPENAIRE_SITE, CFG_ACCESS_CONTROL_LEVEL_SITE class WebInterfaceDisabledPages(WebInterfaceDirectory): @@ -112,12 +109,6 @@ def _lookup(self, component, path): register_exception(alert_admin=True, subject='EMERGENCY') WebInterfaceUnAPIPages = WebInterfaceDumbPages -try: - from invenio.websearch_webinterface import WebInterfaceYourSearchesPages -except: - register_exception(alert_admin=True, subject='EMERGENCY') - WebInterfaceYourSearchesPages = WebInterfaceDumbPages - try: from invenio.bibdocfile_webinterface import bibdocfile_legacy_getfile except: @@ -310,58 +301,81 @@ def _lookup(self, component, path): WebInterfaceAuthorlistPages = WebInterfaceDumbPages try: - from invenio.bibencode_youtube import WebInterfaceYoutube + from invenio.webnews_webinterface import WebInterfaceWebNewsPages except: register_exception(alert_admin=True, subject='EMERGENCY') - WebInterfaceYoutube = WebInterfaceDumbPages + WebInterfaceWebNewsPages = WebInterfaceDumbPages try: - from invenio.webnews_webinterface import WebInterfaceWebNewsPages + from invenio.bibmedia_webinterface import WebInterfaceAlbum except: register_exception(alert_admin=True, subject='EMERGENCY') - WebInterfaceWebNewsPages = WebInterfaceDumbPages + WebInterfaceAlbum = WebInterfaceDumbPages +try: + from invenio.mediaarchive_webinterface import WebInterfaceMediaArchiveErrorHandler +except: + register_exception(alert_admin=True, subject='EMERGENCY') + WebInterfaceMediaArchiveErrorHandler = WebInterfaceDumbPages -if CFG_CERN_SITE: - try: - from invenio.aleph_webinterface import WebInterfaceAlephPages - except: - register_exception(alert_admin=True, subject='EMERGENCY') - WebInterfaceAlephPages = WebInterfaceDumbPages +###CDS HACK### +try: + from invenio.aleph_webinterface import WebInterfaceAlephPages +except: + register_exception(alert_admin=True, subject='EMERGENCY') + WebInterfaceAlephPages = WebInterfaceDumbPages - try: - from invenio.setlink_webinterface import WebInterfaceSetLinkPages - except: - register_exception(alert_admin=True, subject='EMERGENCY') - WebInterfaceSetLinkPages = WebInterfaceDumbPages +try: + from invenio.setlink_webinterface import WebInterfaceSetLinkPages +except: + register_exception(alert_admin=True, subject='EMERGENCY') + WebInterfaceSetLinkPages = WebInterfaceDumbPages - try: - from invenio.yellowreports_webinterface import WebInterfaceYellowReportsPages - except: - register_exception(alert_admin=True, subject='EMERGENCY') - WebInterfaceYellowReportsPages = WebInterfaceDumbPages +try: + from invenio.yellowreports_webinterface import WebInterfaceYellowReportsPages +except: + register_exception(alert_admin=True, subject='EMERGENCY') + WebInterfaceYellowReportsPages = WebInterfaceDumbPages - try: - from invenio.webimages_webinterface import WebInterfaceImagesPages - except: - register_exception(alert_admin=True, subject='EMERGENCY') - WebInterfaceImagesPages = WebInterfaceDumbPages +try: + from invenio.webimages_webinterface import WebInterfaceImagesPages +except: + register_exception(alert_admin=True, subject='EMERGENCY') + WebInterfaceImagesPages = WebInterfaceDumbPages - try: - from invenio.embedvideo_webinterface import WebInterfaceEmbedVideo - except: - register_exception(alert_admin=True, subject='EMERGENCE') - WebInterfaceEmbedVideo = WebInterfaceDumbPages +try: + from invenio.embedvideo_webinterface import WebInterfaceEmbedVideo +except: + register_exception(alert_admin=True, subject='EMERGENCE') + WebInterfaceEmbedVideo = WebInterfaceDumbPages - try: - from invenio.webapi_webinterface import WebInterfaceAPIPages - except: - register_exception(alert_admin=True, subject='EMERGENCE') - WebInterfaceAPIPages = WebInterfaceDumbPages +try: + from invenio.webapi_webinterface import WebInterfaceAPIPages +except: + register_exception(alert_admin=True, subject='EMERGENCE') + WebInterfaceAPIPages = WebInterfaceDumbPages + +try: + from invenio.webcomment_webinterface import WebInterfaceYourCommentsPages +except: + register_exception(alert_admin=True, subject='EMERGENCY') + WebInterfaceYourAlertsPages = WebInterfaceDumbPages + +try: + from invenio.bibencode_youtube import WebInterfaceYoutube +except: + register_exception(alert_admin=True, subject='EMERGENCY') + WebInterfaceYoutube = WebInterfaceDumbPages + +try: + from invenio.feedback_webinterface import WebInterfaceFeedback +except: + register_exception(alert_admin=True, subject='EMERGENCY') + WebInterfaceFeedback = WebInterfaceDumbPages + +cds_exports = ['cdslib', 'setlink', 'images', 'video', 'api', 'yellowrep'] +###END CDS HACK### - cds_exports = ['cdslib', 'setlink', 'images', 'video', 'api', 'yellowrep'] -else: - cds_exports = [] if CFG_OPENAIRE_SITE: try: @@ -407,7 +421,6 @@ class WebInterfaceInvenio(WebInterfaceSearchInterfacePages): 'yourcomments', 'ill', 'yourgroups', - 'yoursearches', 'yourtickets', 'comments', 'error', @@ -432,8 +445,11 @@ class WebInterfaceInvenio(WebInterfaceSearchInterfacePages): 'goto', 'info', 'authorlist', - 'youtube', 'news', + 'media', + 'mediaarchive', + 'youtube', + 'feedback', ] + test_exports + openaire_exports + cds_exports def __init__(self): @@ -452,7 +468,6 @@ def __init__(self): yourloans = WebInterfaceDisabledPages() ill = WebInterfaceDisabledPages() yourgroups = WebInterfaceDisabledPages() - yoursearches = WebInterfaceDisabledPages() yourtickets = WebInterfaceDisabledPages() comments = WebInterfaceDisabledPages() error = WebInterfaceErrorPages() @@ -476,15 +491,19 @@ def __init__(self): yourcomments = WebInterfaceDisabledPages() goto = WebInterfaceDisabledPages() authorlist = WebInterfaceDisabledPages() - youtube = WebInterfaceYoutube() news = WebInterfaceDisabledPages() - if CFG_CERN_SITE: - cdslib = WebInterfaceDisabledPages() - setlink = WebInterfaceDisabledPages() - yellowrep = WebInterfaceYellowReportsPages() - images = WebInterfaceImagesPages() - video = WebInterfaceEmbedVideo() - api = WebInterfaceAPIPages() + media = WebInterfaceAlbum() + mediaarchive = WebInterfaceMediaArchiveErrorHandler() + youtube = WebInterfaceYoutube() + feedback = WebInterfaceFeedback() + ###CDS HACK + cdslib = WebInterfaceAlephPages() + setlink = WebInterfaceSetLinkPages() + yellowrep = WebInterfaceYellowReportsPages() + images = WebInterfaceImagesPages() + video = WebInterfaceEmbedVideo() + api = WebInterfaceAPIPages() + ##END CDS HACK else: submit = WebInterfaceSubmitPages() youraccount = WebInterfaceYourAccountPages() @@ -494,7 +513,6 @@ def __init__(self): yourloans = WebInterfaceYourLoansPages() ill = WebInterfaceILLPages() yourgroups = WebInterfaceYourGroupsPages() - yoursearches = WebInterfaceYourSearchesPages() yourtickets = WebInterfaceYourTicketsPages() comments = WebInterfaceCommentsPages() error = WebInterfaceErrorPages() @@ -518,15 +536,20 @@ def __init__(self): yourcomments = WebInterfaceYourCommentsPages() goto = WebInterfaceGotoPages() authorlist = WebInterfaceAuthorlistPages() - youtube = WebInterfaceYoutube() news = WebInterfaceWebNewsPages() - if CFG_CERN_SITE: - cdslib = WebInterfaceAlephPages() - setlink = WebInterfaceSetLinkPages() - yellowrep = WebInterfaceYellowReportsPages() - images = WebInterfaceImagesPages() - video = WebInterfaceEmbedVideo() - api = WebInterfaceAPIPages() + media = WebInterfaceAlbum() + mediaarchive = WebInterfaceMediaArchiveErrorHandler() + youtube = WebInterfaceYoutube() + feedback = WebInterfaceFeedback() + ###CDS HACK + cdslib = WebInterfaceAlephPages() + setlink = WebInterfaceSetLinkPages() + yellowrep = WebInterfaceYellowReportsPages() + images = WebInterfaceImagesPages() + video = WebInterfaceEmbedVideo() + api = WebInterfaceAPIPages() + ##END CDS HACK + # This creates the 'handler' function, which will be invoked directly # by mod_python. From a317f91a55d8a3074d5b5b92f8363a256ffb9af8 Mon Sep 17 00:00:00 2001 From: Sebastian Witowski Date: Mon, 26 Sep 2016 15:45:18 +0200 Subject: [PATCH 58/59] BibSword: Move tables creation to tabcreate.sql Signed-off-by: Sebastian Witowski --- .../bibsword/lib/bibsword_client_dblayer.py | 35 ------------------- modules/miscutil/sql/tabcreate.sql | 33 +++++++++++++++++ 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/modules/bibsword/lib/bibsword_client_dblayer.py b/modules/bibsword/lib/bibsword_client_dblayer.py index 4cf43e4a18..10d8dd117b 100644 --- a/modules/bibsword/lib/bibsword_client_dblayer.py +++ b/modules/bibsword/lib/bibsword_client_dblayer.py @@ -26,14 +26,6 @@ from invenio.dbquery import run_sql -# CREATE TABLE `swrCLIENTTEMPSUBMISSION` ( -# `id` varchar(128) NOT NULL, -# `object` longblob NOT NULL, -# `last_updated` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', -# PRIMARY KEY (`id`) -# ) - - def store_temp_submission( sid, sobject_blob @@ -124,19 +116,6 @@ def delete_temp_submission( return res -# CREATE TABLE `swrCLIENTSUBMISSION` ( -# `id_user` int(15) unsigned NOT NULL, -# `id_record` mediumint(8) unsigned NOT NULL, -# `id_server` int(11) unsigned NOT NULL, -# `url_alternate` varchar(256) NOT NULL DEFAULT '', -# `url_edit` varchar(256) NOT NULL DEFAULT '', -# `status` varchar(256) NOT NULL DEFAULT '', -# `submitted` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', -# `last_updated` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', -# PRIMARY KEY (`id_record`,`id_server`) -# ) - - def archive_submission( user_id, record_id, @@ -243,20 +222,6 @@ def get_submissions( return result -# CREATE TABLE `swrCLIENTSERVER` ( -# `id` int(11) unsigned NOT NULL AUTO_INCREMENT, -# `name` varchar(64) NOT NULL, -# `engine` varchar(64) NOT NULL, -# `username` varchar(64) NOT NULL, -# `password` varchar(64) NOT NULL, -# `email` varchar(64) NOT NULL, -# `service_document_parsed` longblob, -# `update_frequency` varchar(16) NOT NULL, -# `last_updated` timestamp DEFAULT 0, -# PRIMARY KEY (`id`) -# ) - - def get_servers( server_id=None, with_dict=False diff --git a/modules/miscutil/sql/tabcreate.sql b/modules/miscutil/sql/tabcreate.sql index 2126a4d4dc..a2514748de 100644 --- a/modules/miscutil/sql/tabcreate.sql +++ b/modules/miscutil/sql/tabcreate.sql @@ -5021,6 +5021,39 @@ CREATE TABLE `pidLOG` ( CONSTRAINT `pidlog_ibfk_1` FOREIGN KEY (`id_pid`) REFERENCES `pidSTORE` (`id`) ) ENGINE=MYISAM; +-- tables for bibsword +CREATE TABLE `swrCLIENTTEMPSUBMISSION` ( + `id` varchar(128) NOT NULL, + `object` longblob NOT NULL, + `last_updated` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', + PRIMARY KEY (`id`) +) ENGINE=MYISAM; + +CREATE TABLE `swrCLIENTSUBMISSION` ( + `id_user` int(15) unsigned NOT NULL, + `id_record` mediumint(8) unsigned NOT NULL, + `id_server` int(11) unsigned NOT NULL, + `url_alternate` varchar(256) NOT NULL DEFAULT '', + `url_edit` varchar(256) NOT NULL DEFAULT '', + `status` varchar(256) NOT NULL DEFAULT '', + `submitted` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', + `last_updated` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', + PRIMARY KEY (`id_record`,`id_server`) +) ENGINE=MYISAM; + +CREATE TABLE `swrCLIENTSERVER` ( + `id` int(11) unsigned NOT NULL AUTO_INCREMENT, + `name` varchar(64) NOT NULL, + `engine` varchar(64) NOT NULL, + `username` varchar(64) NOT NULL, + `password` varchar(64) NOT NULL, + `email` varchar(64) NOT NULL, + `service_document_parsed` longblob, + `update_frequency` varchar(16) NOT NULL, + `last_updated` timestamp DEFAULT 0, + PRIMARY KEY (`id`) +) ENGINE=MYISAM; + -- maint-1.1 upgrade recipes: INSERT INTO upgrade (upgrade, applied) VALUES ('invenio_release_1_1_0',NOW()); INSERT INTO upgrade (upgrade, applied) VALUES ('invenio_2012_10_31_tablesorter_location',NOW()); From 713d05b7297334cd85848324dfa41944dd28ca52 Mon Sep 17 00:00:00 2001 From: Martin Vesper Date: Tue, 27 Sep 2016 15:14:02 +0200 Subject: [PATCH 59/59] Bibcirculation: pending request list * Changes the sql query in *get_loan_request_by_status* to also display those request with expired *period_of_interest_to* fields. --- modules/bibcirculation/lib/bibcirculation_dblayer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/bibcirculation/lib/bibcirculation_dblayer.py b/modules/bibcirculation/lib/bibcirculation_dblayer.py index 3141505563..f4f08f11fb 100644 --- a/modules/bibcirculation/lib/bibcirculation_dblayer.py +++ b/modules/bibcirculation/lib/bibcirculation_dblayer.py @@ -345,7 +345,6 @@ def get_loan_request_by_status(status): WHERE lr.id_crcBORROWER=bor.id AND it.barcode=lr.barcode AND lib.id = it.id_crcLIBRARY AND lr.status=%s AND lr.period_of_interest_from <= NOW() - AND lr.period_of_interest_to >= NOW() ORDER BY lr.request_date""" res = run_sql(query , (status, ))