From fca1d59a73d48fac0bc2eff4e02c4979d9f55962 Mon Sep 17 00:00:00 2001 From: "John Connor (Mulligan)" Date: Mon, 13 Feb 2023 09:46:21 -0600 Subject: [PATCH 01/33] captains and owners directly from voyages table --- .../commands/import_texas_feb7_2023.py | 244 +++++++++++------- 1 file changed, 145 insertions(+), 99 deletions(-) diff --git a/voyages/apps/past/management/commands/import_texas_feb7_2023.py b/voyages/apps/past/management/commands/import_texas_feb7_2023.py index 3a8c82256..d3f454745 100644 --- a/voyages/apps/past/management/commands/import_texas_feb7_2023.py +++ b/voyages/apps/past/management/commands/import_texas_feb7_2023.py @@ -7,7 +7,7 @@ import csv from voyages.apps.common.utils import * from voyages.apps.past.models import Enslaved, EnslavedInRelation, EnslavedSourceConnection, EnslavementRelation, EnslavementRelationType, EnslaverAlias, EnslaverIdentity, EnslaverIdentitySourceConnection, EnslaverInRelation, EnslaverRole, EnslaverVoyageConnection, EnslaverCachedProperties,CaptiveFate -from voyages.apps.voyage.models import Place, Voyage,VoyageSources +from voyages.apps.voyage.models import Place, Voyage,VoyageSources,VoyageShipOwnerConnection,VoyageCaptainConnection import re class Command(BaseCommand): @@ -17,7 +17,6 @@ def handle(self, *args, **options): def noblank(v,c=None,blank=False): if type(v)==list: v=v[0] - if v in ['',None]: if blank: return '' @@ -32,70 +31,144 @@ def noblank(v,c=None,blank=False): return v def getuniqueorcreatenew(uniqueobj,objdata): - if len(uniqueobj)==1: + if len(uniqueobj)!=0: return uniqueobj[0] - elif len(uniqueobj)==0: - return objdata else: - print("unique failed. quitting.",uniqueobj) - - + return objdata - ##step 1: enslavers (aliases and names mapped) - ##THIS IS VERY SIMPLE RIGHT NOW. IT ASSUMES UNIQUE NAMES FOR ALL ENSLAVERS WHOSE ALIASES HAVE "TEXAS" IN THEIR MANUAL ID FIELD. - ##IT DOES NOT WRITE NEW DATA TO THESE ENSLAVERS IF THEY EXIST (BECAUSE OUR BIOGRAPHICAL DATA ON THEM RIGHT NOW IS MINIMAL) - ##BUT IT DOES CREATE NEW ALIAS AND IDENTITY OBJECTS AS NEEDED ON THE BASIS OF THE UNIQUE NAME CONSTRAING -- AND THEN LINK THEM + allvoyages=Voyage.all_dataset_objects.all().filter(dataset=1) + + ##step 1: enslaver setup + enslavercachedproperties=[] + enslavervoyageconnections=[] + enslaver_identities={} + enslaver_aliases={} + enslaver_identity_pk_ai=max(EnslaverIdentity.objects.all().values_list('id'))[0]+1 + enslaver_alias_pk_ai=max(EnslaverAlias.objects.all().values_list('id'))[0]+1 + texas_enslaver_aliases=EnslaverAlias.objects.all().filter(manual_id__icontains='Texas') + texas_enslaver_identities=EnslaverIdentity.objects.all().filter(aliases__manual_id__icontains='Texas') enslaver_roles={} - - for enslaver_role_name in ['Owner','Shipper','Captain']: - + for enslaver_role_name in ['Owner','Shipper','Captain','Investor']: enslaver_role=getuniqueorcreatenew( EnslaverRole.objects.all().filter(name=enslaver_role_name), EnslaverRole(name=enslaver_role_name) ) - enslaver_roles[enslaver_role_name]=enslaver_role - enslaver_identities={} - enslaver_aliases={} - enslavercachedproperties=[] - enslavervoyageconnections=[] - enslaver_identity_pk_ai=max(EnslaverIdentity.objects.all().values_list('id'))[0]+1 - enslaver_alias_pk_ai=max(EnslaverAlias.objects.all().values_list('id'))[0]+1 + ##step 1A: captains and vessel owners ('investors') from the voyages table, corresponding with the voyages in this sheet + with open('voyages/apps/past/management/commands/texas_enslaved.csv',encoding='utf-8-sig') as csvfile: + reader = csv.DictReader(csvfile) + sheet_voyage_ids=[] + for row in reader: + voyage_id=noblank(row['VOYAGEID'],"int") + sheet_voyage_ids.append(voyage_id) + + sheet_voyage_ids=list(set(sheet_voyage_ids)) + + sheet_voyages=[noblank(list(v for v in allvoyages.filter(voyage_id=vi))) for vi in sheet_voyage_ids] + shipownernames={} + shipcaptainnames={} + + for voy in sheet_voyages: + shipowners=[vsoc.owner for vsoc in VoyageShipOwnerConnection.objects.all().filter(voyage=voy) if vsoc.owner.name is not None] + shipcaptains=[vcc.captain for vcc in VoyageCaptainConnection.objects.all().filter(voyage=voy) if vcc.captain.name is not None] + for shipowner in shipowners: + shipownername=shipowner.name + if shipownername in shipownernames: + shipownernames[shipownername].append(voy) + else: + shipownernames[shipownername]=[voy] + for shipcaptain in shipcaptains: + shipcaptainname=shipcaptain.name + if shipcaptainname in shipcaptainnames: + shipcaptainnames[shipcaptainname].append(voy) + else: + shipcaptainnames[shipcaptainname]=[voy] + + role=enslaver_roles['Investor'] + for enslavername in shipownernames: + enslaver_identity=EnslaverIdentity( + id=enslaver_identity_pk_ai, + principal_alias=enslavername + ) + enslaver_identity.save() + enslaver_alias=EnslaverAlias( + id=enslaver_alias_pk_ai, + identity=enslaver_identity, + alias=enslavername, + manual_id="Texas_"+str(enslaver_alias_pk_ai) + ) + enslaver_alias.save() + + ecp=EnslaverCachedProperties(identity=enslaver_identity,enslaved_count=0) + ecp.save() + enslaver_identity_pk_ai+=1 + enslaver_alias_pk_ai+=1 + for voy in shipownernames[enslavername]: + enslaver_voyage_connection=EnslaverVoyageConnection( + enslaver_alias=enslaver_alias, + role=role, + voyage=voy + ) + enslaver_voyage_connection.save() + enslavervoyageconnections.append(enslaver_voyage_connection) + + role=enslaver_roles['Captain'] + for enslavername in shipcaptainnames: + enslaver_identity=EnslaverIdentity( + id=enslaver_identity_pk_ai, + principal_alias=enslavername + ) + enslaver_identity.save() + enslaver_alias=EnslaverAlias( + id=enslaver_alias_pk_ai, + identity=enslaver_identity, + alias=enslavername, + manual_id="Texas_"+str(enslaver_alias_pk_ai) + ) + enslaver_alias.save() + + ecp=EnslaverCachedProperties(identity=enslaver_identity,enslaved_count=0) + ecp.save() + enslaver_identity_pk_ai+=1 + enslaver_alias_pk_ai+=1 + for voy in shipcaptainnames[enslavername]: + enslaver_voyage_connection=EnslaverVoyageConnection( + enslaver_alias=enslaver_alias, + role=role, + voyage=voy + ) + enslaver_voyage_connection.save() + enslavervoyageconnections.append(enslaver_voyage_connection) + + print("%d captain names" %len(shipcaptainnames)) + print("%d captain & owner aliases" %len(shipownernames)) + print("%d captain & owner voyage connections" %len(enslavervoyageconnections)) + + ##step 1B: enslavers with direct enslavement connections + ##THIS IS VERY SIMPLE RIGHT NOW. IT ASSUMES UNIQUE NAMES FOR ALL ENSLAVERS WHOSE ALIASES HAVE "TEXAS" IN THEIR MANUAL ID FIELD. + ##IT DOES NOT WRITE NEW DATA TO THESE ENSLAVERS IF THEY EXIST (BECAUSE OUR BIOGRAPHICAL DATA ON THEM RIGHT NOW IS MINIMAL) + ##BUT IT DOES CREATE NEW ALIAS AND IDENTITY OBJECTS AS NEEDED ON THE BASIS OF THE UNIQUE NAME CONSTRAING -- AND THEN LINK THEM + - allvoyages=Voyage.all_dataset_objects.all().filter(dataset=1) - with open('voyages/apps/past/management/commands/texas_enslaved.csv',encoding='utf-8-sig') as csvfile: reader = csv.DictReader(csvfile) enslavernames=[] captains_voyages={} - enslavercolumnnames=['Captain A','Shipper','Owner A','Owner B'] + enslavercolumnnames=['Shipper','Owner A','Owner B'] + + for row in reader: for ecn in enslavercolumnnames: enslavernames.append(row[ecn]) - if ecn=='Captain A': - captain_name=row[ecn] - voyage_id=noblank(row['VOYAGEID'],"int") - voyage=noblank(list(v for v in allvoyages.filter(voyage_id=voyage_id))) - if captain_name in captains_voyages: - if voyage not in captains_voyages[captain_name]: - captains_voyages[captain_name].append(voyage) - else: - captains_voyages[captain_name]=[voyage] - - ###along the way, create name-keyed dictionaries for the enslavers' identities + enslavernames=[i for i in list(set(enslavernames)) if i!=''] # captain_names=[i for i in list(set(captain_names)) if i!=''] - texas_enslaver_aliases=EnslaverAlias.objects.all().filter(manual_id__icontains='Texas') - texas_enslaver_identities=EnslaverIdentity.objects.all().filter(aliases__manual_id__icontains='Texas') - - for enslavername in enslavernames: - enslaver_identity=getuniqueorcreatenew( texas_enslaver_identities.filter(aliases__alias=enslavername), EnslaverIdentity( @@ -113,32 +186,8 @@ def getuniqueorcreatenew(uniqueobj,objdata): manual_id="Texas_"+str(enslaver_alias_pk_ai) ) ) - ecp=EnslaverCachedProperties(identity=enslaver_identity,enslaved_count=0) enslavercachedproperties.append(ecp) - - if enslavername in captains_voyages: - captain_voyages=captains_voyages[enslavername] - for voyage in captain_voyages: - - enslaver_voyage_connection=getuniqueorcreatenew( - EnslaverVoyageConnection.objects.all().filter( - enslaver_alias=enslaver_alias, - role=enslaver_roles['Captain'], - voyage=voyage), - EnslaverVoyageConnection( - enslaver_alias=enslaver_alias, - role=enslaver_roles['Captain'], - voyage=voyage - ) - ) - enslavervoyageconnections.append(enslaver_voyage_connection) - - - - - - enslaver_aliases[enslavername]=enslaver_alias enslaver_identities[enslavername]=enslaver_identity @@ -146,13 +195,13 @@ def getuniqueorcreatenew(uniqueobj,objdata): enslaver_alias_pk_ai+=1 print("%d enslaver identities" %len(enslaver_identities)) print("%d enslaver aliases" %len(enslaver_aliases)) - + ##step 2: sources sourcecolumnnames=['SOURCEA','SOURCEB'] sources={} with open('voyages/apps/past/management/commands/texas_enslaved.csv',encoding='utf-8-sig') as csvfile: - tmp_sources={} + tmp_sources=[] reader = csv.DictReader(csvfile) #very basic -- the first letters before the comma. #Erik Engquist: when you think you've solved a problem with regular expressions, you've got three problems @@ -163,20 +212,20 @@ def getuniqueorcreatenew(uniqueobj,objdata): entry=row[scn] if entry!='': shortref=re.search("[^,]+",row[scn]).group(0) - leftover=row[scn][len(shortref)+1:] - tmp_sources[shortref]=leftover + tmp_sources.append(shortref) + tmp_sources=list(set(tmp_sources)) db_sources=VoyageSources.objects.all() - for source in tmp_sources: + for short_ref in tmp_sources: this_source=getuniqueorcreatenew( - db_sources.filter(short_ref=source), + db_sources.filter(short_ref=short_ref), VoyageSources( - short_ref=source, + short_ref=short_ref, source_type_id=1, - full_ref=source + full_ref=short_ref ) ) # print(this_source) - sources[source]={'obj':this_source,'text_ref':tmp_sources[source]} + sources[short_ref]=this_source print("%d sources" %len(sources)) @@ -191,19 +240,21 @@ def getuniqueorcreatenew(uniqueobj,objdata): reader = csv.DictReader(csvfile) enslaved=Enslaved.objects.all() - - for row in reader: - enslavement_sources_shortrefs=list(set([re.search("[^,]+",row[i]).group(0) for i in ['SOURCEA','SOURCEB'] if row[i]!=''])) - - enslaved_sources=[sources[i] for i in enslavement_sources_shortrefs] + for row in reader: + enslavement_sources=[row[i] for i in sourcecolumnnames if row[i]!=''] + tmp_sources={} + for enslavement_source in enslavement_sources: + shortref=re.search("[^,]+",enslavement_source).group(0) + leftover=enslavement_source[len(shortref)+2:] + tmp_sources[shortref]={"obj":sources[shortref],"text_ref":leftover} captive_fate_id=row['Captive fate'] if captive_fate_id != '': - + captive_fate_id=int(captive_fate_id) captive_fate_name=captivefatenamesdict.get(captive_fate_id) or "Unknown Value" - + captive_fate=getuniqueorcreatenew( CaptiveFate.objects.all().filter(id=captive_fate_id), CaptiveFate( @@ -212,7 +263,7 @@ def getuniqueorcreatenew(uniqueobj,objdata): ) ) captivefates[captive_fate_id]=captive_fate - + else: captive_fate=None @@ -244,8 +295,8 @@ def getuniqueorcreatenew(uniqueobj,objdata): c=1 - for source in enslaved_sources: - + for shortref in tmp_sources: + source=tmp_sources[shortref] enslavedsourceconnection=EnslavedSourceConnection( enslaved=enslaved_person, source=source['obj'], @@ -357,21 +408,15 @@ def getuniqueorcreatenew(uniqueobj,objdata): ) enslavedinrelation_pk_ai+=1 enslaved_in_relations.append(enslaved_in_relation) - - - - - sources2={sources[source]['obj'] for source in sources} print("%d enslavement relations" %len(enslavement_relations)) print("%d enslaver in relations" %len(enslaver_in_relations)) print("%d enslaved in relations" %len(enslaved_in_relations)) - print("%d enslaver voyage connections" %len(enslavervoyageconnections)) print("%d enslaver roles" %len(enslaver_roles)) - + itemlists=[ captivefates, - sources2, +# sources, enslaveds, enslavedsourceconnections, enslavement_relations, @@ -380,24 +425,25 @@ def getuniqueorcreatenew(uniqueobj,objdata): enslaver_aliases, enslaver_in_relations, enslaved_in_relations, - enslavervoyageconnections, enslavercachedproperties ] - + for itemlist in itemlists: - - if type(itemlist)==list: + + if type(itemlist)==list and len(itemlist)>0: print("importing objects like this-->",itemlist[0]) for item in itemlist: try: item.save() except: print(item.__dict__) - - elif type(itemlist)==dict: + + elif type(itemlist)==dict and len(itemlist)>0: print("importing objects like this-->",itemlist[list(itemlist.keys())[0]]) for k in itemlist: try: itemlist[k].save() except: - print(print(k,itemlist[k].__dict__)) \ No newline at end of file + print(print(k,itemlist[k].__dict__)) + + From 3cb6923e7780135d57bbb9073cb95fad400dc183 Mon Sep 17 00:00:00 2001 From: Domingos Date: Fri, 10 Feb 2023 16:00:28 -0300 Subject: [PATCH 02/33] Enslaver role search. Backend and frontend implementation. --- voyages/apps/past/models.py | 16 +++++++++++- .../templates/past-enslavers/_itinerary.html | 5 ++++ voyages/apps/past/urls.py | 3 +++ voyages/apps/past/views.py | 8 +++++- .../sitemedia/scripts/vue/enslavers/app.js | 9 ------- .../scripts/vue/enslavers/includes/helpers.js | 26 ++++++------------- voyages/sitemedia/scripts/vue/past/app.js | 9 ------- .../vue/variables/past-enslavers/itinerary.js | 13 ++++++++++ 8 files changed, 51 insertions(+), 38 deletions(-) diff --git a/voyages/apps/past/models.py b/voyages/apps/past/models.py index 8ec7fe998..e2ff581b7 100644 --- a/voyages/apps/past/models.py +++ b/voyages/apps/past/models.py @@ -482,7 +482,11 @@ def get_year(s): props = EnslaverCachedProperties() props.identity_id = id props.enslaved_count = item.get('enslaved_count', 0) - props.roles = ','.join([str(role) for role in item.get('roles', [])]) + # Note: we leave a comma after each number *on purpose*. This is so + # we can search for roles by querying for contains "{role_id},". + # Without the comma in the end, we could get false positives for ids + # with multiple digits. + props.roles = ''.join(sorted(f"{role}," for role in item.get('roles', []))) props.transactions_amount = item.get('tot_amount', 0) props.first_year = item.get('min_year', None) props.last_year = item.get('max_year', None) @@ -1388,6 +1392,7 @@ def __init__(self, voyage_id=None, source=None, enslaved_count=None, + roles=None, voyage_datasets=None, order_by=None): """ @@ -1411,6 +1416,8 @@ def __init__(self, full_ref @param: enslaved_count A pair (a, b) such that the enslaver has an associated enslaved count between a and b. + @param: roles a list of role pks that should be matched (an enslaver may + appear in multiple roles). @param: order_by An array of dicts { 'columnName': 'NAME', 'direction': 'ASC or DESC' }. Note that if the search is fuzzy, then the fallback value of @@ -1426,6 +1433,7 @@ def __init__(self, self.voyage_id = voyage_id self.source = source self.enslaved_count = enslaved_count + self.roles = roles self.voyage_datasets = voyage_datasets self.order_by = order_by or [{'columnName': 'pk', 'direction': 'asc'}] @@ -1485,6 +1493,12 @@ def add_voyage_field(q, field, op, val): q = add_voyage_field(q, 'voyage_dates__imp_arrival_at_port_of_dis', 'range', _year_range_conv(self.year_range)) if self.enslaved_count: q = q.filter(cached_properties__enslaved_count__range=self.enslaved_count) + if self.roles: + terms = None + for pk in self.roles: + term = Q(cached_properties__roles__contains=f"{pk},") + terms = (term | terms) if terms else term + q = q.filter(terms) if self.voyage_datasets is not None: bitvec = 0 for x in self.voyage_datasets: diff --git a/voyages/apps/past/templates/past-enslavers/_itinerary.html b/voyages/apps/past/templates/past-enslavers/_itinerary.html index fc59081ae..f41caa052 100644 --- a/voyages/apps/past/templates/past-enslavers/_itinerary.html +++ b/voyages/apps/past/templates/past-enslavers/_itinerary.html @@ -29,6 +29,11 @@ :filter="scope.filters.var_enslaved_count"> + + + diff --git a/voyages/apps/past/urls.py b/voyages/apps/past/urls.py index cda99c10b..ce9f658b8 100644 --- a/voyages/apps/past/urls.py +++ b/voyages/apps/past/urls.py @@ -16,6 +16,9 @@ url(r'^api/language-groups', voyages.apps.past.views.get_language_groups, name='language-groups'), + url(r'^api/enslaver-roles', + voyages.apps.past.views.get_enslaver_roles, + name='enslaver-roles'), url(r'^database/(?P[\w\-]+)', voyages.apps.past.views.enslaved_database, name='database'), diff --git a/voyages/apps/past/views.py b/voyages/apps/past/views.py index 547c68432..8c2515896 100644 --- a/voyages/apps/past/views.py +++ b/voyages/apps/past/views.py @@ -24,7 +24,7 @@ from voyages.apps.common.views import get_filtered_results from .models import (AltLanguageGroupName, Enslaved, EnslavedContribution, EnslavedContributionLanguageEntry, - EnslavedContributionNameEntry, EnslavedContributionStatus, EnslavedInRelation, EnslavedSearch, EnslavementRelation, EnslaverContribution, EnslaverInRelation, EnslaverSearch, EnslaverVoyageConnection, + EnslavedContributionNameEntry, EnslavedContributionStatus, EnslavedInRelation, EnslavedSearch, EnslavementRelation, EnslaverContribution, EnslaverInRelation, EnslaverRole, EnslaverSearch, EnslaverVoyageConnection, LanguageGroup, MultiValueHelper, ModernCountry, EnslavedNameSearchCache, _modern_name_fields, _name_fields) from voyages.apps.voyage.models import Place,Region @@ -152,6 +152,12 @@ def get_language_groups(request): return JsonResponse([{ "id": item["id"], "name": item["name"], "lat": item["latitude"], "lng": item["longitude"], "alts": item[alt_names_key], "countries": item[countries_list_key] } for item in items], safe=False) + +@csrf_exempt +@cache_page(3600) +def get_enslaver_roles(request): + return JsonResponse([{"id": r.id, "label": r.name} for r in EnslaverRole.objects.all()], safe=False) + @csrf_exempt @cache_page(3600) def get_enumeration(_, model_name): diff --git a/voyages/sitemedia/scripts/vue/enslavers/app.js b/voyages/sitemedia/scripts/vue/enslavers/app.js index 05dacded4..bf81c91e7 100644 --- a/voyages/sitemedia/scripts/vue/enslavers/app.js +++ b/voyages/sitemedia/scripts/vue/enslavers/app.js @@ -45,15 +45,6 @@ var searchBar = new Vue({ }] } }, - enslaverRoles: { - 1: "Captain", - 2: "Investor", - 3: "Buyer", - 4: "Seller", - 5: "Owner", - 6: "Shipper", - 7: "Consignor", - }, activated: false, saved: [], options: { diff --git a/voyages/sitemedia/scripts/vue/enslavers/includes/helpers.js b/voyages/sitemedia/scripts/vue/enslavers/includes/helpers.js index bc1dbd2a1..134777408 100644 --- a/voyages/sitemedia/scripts/vue/enslavers/includes/helpers.js +++ b/voyages/sitemedia/scripts/vue/enslavers/includes/helpers.js @@ -157,18 +157,6 @@ function processResponse(json, mainDatatable, fuzzySearch) { row.ranking++; } - if (row.enslavers_list) { - var enslaversList = {}; - row.enslavers_list.forEach((value, index) => { - if (enslaversList[value.enslaver_name] === undefined) { - enslaversList[value.enslaver_name] = []; - } - - enslaversList[value.enslaver_name].push(gettext(searchBar.enslaverRoles[value.enslaver_role])); - }); - row.enslavers_list = enslaversList; - } - if (row.alias_list) { var aliasList = {}; aliasList = row.alias_list.filter((element) => { @@ -732,12 +720,12 @@ function loadTreeselectOptions(vm, vTreeselect, filter, callback) { case 'modern_country': var apiUrl = '/past/api/modern-countries'; break; - case 'ethnicity': - var apiUrl = '/past/api/ethnicities'; - break; case 'language_groups': var apiUrl = '/past/api/language-groups'; break; + case 'roles': + var apiUrl = '/past/api/enslaver-roles'; + break; default: callback("Error: varName " + varName + " is not acceptable"); return false; @@ -750,14 +738,16 @@ function loadTreeselectOptions(vm, vTreeselect, filter, callback) { switch (varName) { case 'register_country': case 'modern_country': - var options = parseCountries(response); + options = parseCountries(response); break; case 'ethnicity': - var options = parseEthnicities(response); + options = parseEthnicities(response); break; case 'language_groups': - var options = parseLanguageGroups(response); + options = parseLanguageGroups(response); break; + default: + options = response.data; } vm.filterData.treeselectOptions[varName] = options; diff --git a/voyages/sitemedia/scripts/vue/past/app.js b/voyages/sitemedia/scripts/vue/past/app.js index 2e34b9e01..38582bb3c 100644 --- a/voyages/sitemedia/scripts/vue/past/app.js +++ b/voyages/sitemedia/scripts/vue/past/app.js @@ -53,15 +53,6 @@ var searchBar = new Vue({ }] } }, - enslaverRoles: { - 1: "Captain", - 2: "Investor", - 3: "Buyer", - 4: "Seller", - 5: "Owner", - 6: "Shipper", - 7: "Consignor", - }, activated: false, saved: [], options: { diff --git a/voyages/sitemedia/scripts/vue/variables/past-enslavers/itinerary.js b/voyages/sitemedia/scripts/vue/variables/past-enslavers/itinerary.js index 4f8ff810f..2319ccb56 100644 --- a/voyages/sitemedia/scripts/vue/variables/past-enslavers/itinerary.js +++ b/voyages/sitemedia/scripts/vue/variables/past-enslavers/itinerary.js @@ -49,6 +49,18 @@ var_enslaved_count = new NumberVariable({ isAdvanced: false }); +var_roles = new TreeselectVariable({ + varName: "roles", + label: gettext("Enslaver Roles"), + description: "", + },{ + op: "is one of", + searchTerm: [], + },{ + isImputed: false, + isAdvanced: false, + }); + var_voyage_datasets = new TreeselectVariable({ varName: "voyage_datasets", label: gettext("Voyages Dataset"), @@ -69,6 +81,7 @@ itinerary = { var_year_range: var_year_range, var_enslaved_count: var_enslaved_count, var_voyage_datasets: var_voyage_datasets, + var_roles: var_roles, count: { changed: 0, From 2dd72a2623011d116133e0f4c314f1ca8ff661fa Mon Sep 17 00:00:00 2001 From: Domingos Date: Tue, 14 Feb 2023 11:26:57 -0300 Subject: [PATCH 03/33] Removing country from Enslaved and its contribution. Enslaver role search. Backend and frontend implementation. Removing country from Enslaved and its contribution. --- .../templates/contribute/review_origins.html | 23 ++----------------- voyages/apps/contribute/views.py | 7 ++---- .../migrations/0027_auto_20230214_1408.py | 23 +++++++++++++++++++ voyages/apps/past/models.py | 6 +---- voyages/apps/past/views.py | 8 ------- 5 files changed, 28 insertions(+), 39 deletions(-) create mode 100644 voyages/apps/past/migrations/0027_auto_20230214_1408.py diff --git a/voyages/apps/contribute/templates/contribute/review_origins.html b/voyages/apps/contribute/templates/contribute/review_origins.html index c6d0d579f..f601be725 100644 --- a/voyages/apps/contribute/templates/contribute/review_origins.html +++ b/voyages/apps/contribute/templates/contribute/review_origins.html @@ -74,13 +74,9 @@

Contributed Language Groups

Propagation

- + @@ -105,7 +101,6 @@

Propagation

{% trans "Embarkation" %} {% trans "Name" %} {% trans "Language Group" %} - {% trans "Country" %} {% trans "Notes" %} @@ -255,14 +250,6 @@ } return data; } - }, { - data: row => ({ pk: row.pk, country: row.modern_country__name }), - render: function(data, type) { - if (type === 'display') { - return ` ${data.country}`; - } - return data; - } }, { data: row => ({ pk: row.pk, notes: row.notes }), render: function(data, type) { @@ -276,7 +263,7 @@ } ], scrollY: '30vh', - paginate: contribData.propagation_candidates.length > 10, + paginate: false, // Use scrolling instead since all the data is loaded upfront anyway info: false, bFilter: false, pageLength: 10, @@ -319,7 +306,6 @@ for (const candidate of contribData.propagation_candidates) { const propName = $(`#check_N${candidate.pk}`).prop('checked'); const propLang = $(`#check_L${candidate.pk}`).prop('checked'); - const propCountry = $(`#check_C${candidate.pk}`).prop('checked'); // Create an action (update or preserve for each propagation field). actions.push(makeAction(propName, candidate, 'modern_name', vmodel.propagatedName)); actions.push(makeAction( @@ -327,11 +313,6 @@ candidate, 'language_group_id', getSelectionId(vmodel.propagatedLanguage))); - actions.push(makeAction( - propCountry && !!vmodel.propagatedCountry, - candidate, - 'modern_country_id', - getSelectionId(vmodel.propagatedCountry))); const notes = $("#notes_" + candidate.pk).text(); if (!!notes || !!candidate.notes) { actions.push(makeAction(true, candidate, 'notes', notes)); diff --git a/voyages/apps/contribute/views.py b/voyages/apps/contribute/views.py index 36836f01a..2a976139a 100644 --- a/voyages/apps/contribute/views.py +++ b/voyages/apps/contribute/views.py @@ -2246,9 +2246,7 @@ def _get_audio_relative_url(name_pk): "contributed_language_groups": [{ "entry_pk": cl.pk, "language_group_pk": cl.language_group.id, - "language_group_name": cl.language_group.name, - "modern_country_pk": cl.modern_country_id, - "modern_country_name": cl.modern_country.name if cl.modern_country else None + "language_group_name": cl.language_group.name } for cl in c.contributed_language_groups.all() ], "notes": c.notes, @@ -2320,8 +2318,7 @@ def create_name_match_clauses(names): .values( \ 'pk', 'gender', 'modern_name', 'documented_name', 'notes', \ 'name_first', 'name_second', 'name_third', \ - 'language_group_id', 'modern_country_id', \ - 'language_group__name', 'modern_country__name', \ + 'language_group_id', 'language_group__name', \ embarkation=F('voyage__voyage_itinerary__imp_principal_place_of_slave_purchase__place')) else: propagation_candidates = [] diff --git a/voyages/apps/past/migrations/0027_auto_20230214_1408.py b/voyages/apps/past/migrations/0027_auto_20230214_1408.py new file mode 100644 index 000000000..8413776fe --- /dev/null +++ b/voyages/apps/past/migrations/0027_auto_20230214_1408.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.17 on 2023-02-14 14:08 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('past', '0026_enslavementrelation_unnamed_enslaved_count'), + ] + + operations = [ + migrations.RemoveField( + model_name='enslaved', + name='modern_country', + ), + migrations.RemoveField( + model_name='enslavedcontributionlanguageentry', + name='modern_country', + ), + ] diff --git a/voyages/apps/past/models.py b/voyages/apps/past/models.py index e2ff581b7..f0c743b5f 100644 --- a/voyages/apps/past/models.py +++ b/voyages/apps/past/models.py @@ -706,10 +706,7 @@ class Enslaved(models.Model): skin_color = models.CharField(max_length=100, null=True, db_index=True) language_group = models.ForeignKey(LanguageGroup, null=True, on_delete=models.CASCADE, - db_index=True) - modern_country = models.ForeignKey(ModernCountry, null=True, default=None, - on_delete=models.CASCADE, - db_index=True) + db_index=True) register_country = models.ForeignKey(RegisterCountry, null=True, on_delete=models.CASCADE, db_index=True) @@ -772,7 +769,6 @@ class EnslavedContributionLanguageEntry(models.Model): related_name='contributed_language_groups') language_group = models.ForeignKey(LanguageGroup, null=True, on_delete=models.CASCADE) - modern_country = models.ForeignKey(ModernCountry, null=True, default=None, on_delete=models.CASCADE) order = models.IntegerField() diff --git a/voyages/apps/past/views.py b/voyages/apps/past/views.py index 8c2515896..38eca47b5 100644 --- a/voyages/apps/past/views.py +++ b/voyages/apps/past/views.py @@ -578,7 +578,6 @@ def enslaved_contribution(request): name_ids.append(name_entry.pk) result['name_ids'] = name_ids language_ids = [] - print("-->",languages) # for i, lang in enumerate(languages): for i in languages: lang_entry = EnslavedContributionLanguageEntry() @@ -591,13 +590,6 @@ def enslaved_contribution(request): transaction.rollback() return HttpResponseBadRequest( 'Invalid language entry in contribution') -# modern_country_id = lang.get('modern_country_id', None) -# lang_entry.modern_country = ModernCountry.objects.get( -# pk=modern_country_id) if modern_country_id else None -# if modern_country_id is None: -# transaction.rollback() -# return HttpResponseBadRequest( -# 'Invalid modern country entry in contribution') lang_entry.save() language_ids.append(lang_entry.pk) result['language_ids'] = language_ids From 19b570ba9a5aa71fad5eacf8c978691039d8a312 Mon Sep 17 00:00:00 2001 From: Domingos Date: Tue, 14 Feb 2023 14:29:59 -0300 Subject: [PATCH 04/33] Adding introduction to the Blog's tag page. --- voyages/apps/blog/admin.py | 17 +++++++++++++++ .../apps/blog/migrations/0002_tag_intro.py | 20 ++++++++++++++++++ voyages/apps/blog/models.py | 3 +++ voyages/apps/blog/templates/blog/index.html | 21 ++++++++++++++++++- voyages/apps/blog/views.py | 20 +++++++++++------- 5 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 voyages/apps/blog/migrations/0002_tag_intro.py diff --git a/voyages/apps/blog/admin.py b/voyages/apps/blog/admin.py index ac53364ad..04ba1d890 100644 --- a/voyages/apps/blog/admin.py +++ b/voyages/apps/blog/admin.py @@ -8,6 +8,8 @@ from django.template.response import TemplateResponse +from voyages.extratools import AdvancedEditor + from .models import Post from .models import Tag from .models import Institution @@ -225,9 +227,24 @@ def news_migration (self, request): actions = [make_published,make_draft,translate_to_new_languages,force_translation] +class TagAdminForm(forms.ModelForm): + """ + Form for editing HTML for FAQ answer + """ + intro = forms.CharField(widget=AdvancedEditor( + attrs={'class': 'tinymcetextarea'})) + + class Meta: + fields = '__all__' + model = Tag + class TagAdmin(admin.ModelAdmin): prepopulated_fields = {'slug':('name',)} + form = TagAdminForm + + + class InstitutionAdmin(admin.ModelAdmin): prepopulated_fields = {'slug':('name',)} diff --git a/voyages/apps/blog/migrations/0002_tag_intro.py b/voyages/apps/blog/migrations/0002_tag_intro.py new file mode 100644 index 000000000..933c3422d --- /dev/null +++ b/voyages/apps/blog/migrations/0002_tag_intro.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.17 on 2023-02-14 14:44 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('blog', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='tag', + name='intro', + field=models.TextField(blank=True), + ), + ] diff --git a/voyages/apps/blog/models.py b/voyages/apps/blog/models.py index 6049fe5db..6fe4677ad 100644 --- a/voyages/apps/blog/models.py +++ b/voyages/apps/blog/models.py @@ -44,6 +44,9 @@ def __str__(self): class Tag(models.Model): name = models.CharField(max_length=200,unique=True) slug = models.SlugField(max_length=200, unique=True) + + intro = models.TextField(blank=True) + class Meta: ordering = ['name'] diff --git a/voyages/apps/blog/templates/blog/index.html b/voyages/apps/blog/templates/blog/index.html index dae294ef3..d4cf21329 100644 --- a/voyages/apps/blog/templates/blog/index.html +++ b/voyages/apps/blog/templates/blog/index.html @@ -33,10 +33,29 @@ {% endblock title %} +{% if intro %} +
+
+ +{% endif %} +
diff --git a/voyages/apps/blog/views.py b/voyages/apps/blog/views.py index a1c3d48fb..a9ff77a14 100644 --- a/voyages/apps/blog/views.py +++ b/voyages/apps/blog/views.py @@ -6,24 +6,30 @@ from django import template register = template.Library() -class PostList(generic.ListView): +class PostList(generic.ListView): template_name = 'blog/index.html' paginate_by = 10 + + base_query = Post.objects.select_related() def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context['title_override'] = self.kwargs.get('title_override') + tag_slug = self.kwargs.get('tag') + if tag_slug: + tag = Tag.objects.get(slug=tag_slug) + if tag and tag.intro: + context['intro'] = tag.intro return context def get_queryset(self): lang_code = self.request.LANGUAGE_CODE or "en" - + q = self.__class__.base_query if self.request.resolver_match.url_name == 'news': - return Post.objects.filter(status=PUBLISH_STATUS, language=lang_code, tags__slug__in=['news']).order_by('-created_on').order_by('-created_on') - elif self.kwargs.get('tag') is None: - return Post.objects.filter(status=PUBLISH_STATUS, language=lang_code).order_by('-created_on').exclude(tags__in = Tag.objects.filter(slug__in = ['author-profile','institution-profile']) ) - - return Post.objects.filter(status=PUBLISH_STATUS, language=lang_code, tags__slug__in=[self.kwargs['tag']]).order_by('-created_on') + return q.filter(status=PUBLISH_STATUS, language=lang_code, tags__slug__in=['news']).order_by('-created_on').order_by('-created_on') + if self.kwargs.get('tag') is None: + return q.filter(status=PUBLISH_STATUS, language=lang_code).order_by('-created_on').exclude(tags__in = Tag.objects.filter(slug__in = ['author-profile','institution-profile']) ) + return q.filter(status=PUBLISH_STATUS, language=lang_code, tags__slug__in=[self.kwargs['tag']]).order_by('-created_on') From 2ee779cdefb884042dc265cd430dc3b4d8957841 Mon Sep 17 00:00:00 2001 From: Domingos Date: Wed, 15 Feb 2023 10:54:13 -0300 Subject: [PATCH 05/33] Fixing issue in get_language_groups request parsing. Fixing access to 'verbose_name' property in field types that do not have them. Fixing percentage filter in helpers.js: mortality rate would send null values and cause the query to return zero results in some cases (probably due to some NaN in the frontend). --- voyages/apps/past/views.py | 8 ++++++-- voyages/apps/voyage/search_views.py | 9 ++++++--- .../sitemedia/scripts/vue/includes/helpers.js | 17 +++++++++++------ 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/voyages/apps/past/views.py b/voyages/apps/past/views.py index 38eca47b5..4ddfd18a1 100644 --- a/voyages/apps/past/views.py +++ b/voyages/apps/past/views.py @@ -130,10 +130,14 @@ def get_modern_countries(_): @csrf_exempt -@cache_page(0) +@cache_page(3600) def get_language_groups(request): #we need a switch between used and unused language groups (search should only have used, contribute should have all) - active_only=json.loads(request.body).get('active_only') + active_only = True + try: + active_only = json.loads(request.body).get('active_only', active_only) + except: + pass countries_list_key = "countries_list" alt_names_key = "alt_names_list" country_helper = MultiValueHelper(countries_list_key, ModernCountry.languages.through, 'languagegroup_id', modern_country_id='moderncountry__pk', country_name='moderncountry__name') diff --git a/voyages/apps/voyage/search_views.py b/voyages/apps/voyage/search_views.py index f194d7e87..58d7ec02e 100644 --- a/voyages/apps/voyage/search_views.py +++ b/voyages/apps/voyage/search_views.py @@ -143,6 +143,7 @@ def perform_search(search, lang): dataset = VoyageDataset.Transatlantic if dataset >= 0: search_terms[u'var_dataset__exact'] = dataset + print(f"search_terms: {search_terms}, custom_terms: {custom_terms}") result = sqs.models(Voyage).filter(**search_terms) for ct in custom_terms: result = result.filter(content=Raw(ct, clean=True)) @@ -659,11 +660,13 @@ def get_download_header(var_name): def follow_field(model, name_to_follow): split = name_to_follow.find('__') current = name_to_follow[:split] if split > 0 else name_to_follow + f = None + result = None try: f = model._meta.get_field(current) - except Exception: - f = None - result = f.verbose_name if f else '' + result = f.verbose_name + except: + pass if split > 0 and f: result += ' ' + \ follow_field(f.remote_field.model, name_to_follow[split + 2:]) diff --git a/voyages/sitemedia/scripts/vue/includes/helpers.js b/voyages/sitemedia/scripts/vue/includes/helpers.js index 0882c82ad..041c4b00f 100644 --- a/voyages/sitemedia/scripts/vue/includes/helpers.js +++ b/voyages/sitemedia/scripts/vue/includes/helpers.js @@ -339,6 +339,14 @@ function resetFilter(filter, group, subGroup) { } } +const safePercentageFilter = (val, fallback) => { + const num = parseInt(val); + if (isNaN(num) || num < 0) { + return fallback; + } + return num * 0.01; +}; + // serialize a filter function serializeFilter(filter) { return JSON.stringify(filter); @@ -459,8 +467,7 @@ function searchAll(filter, filterData) { "PercentageVariable" ) { item["searchTerm"] = - parseInt(filter[key1][key2][key3].value["searchTerm"]) / - 100; + safePercentageFilter(filter[key1][key2][key3].value["searchTerm"], 0); } else { item["searchTerm"] = filter[key1][key2][key3].value["searchTerm"]; @@ -521,11 +528,9 @@ function searchAll(filter, filterData) { "PercentageVariable" ) { var searchTerm0 = - parseInt(filter[key1][key2][key3].value["searchTerm0"]) / - 100; + safePercentageFilter(filter[key1][key2][key3].value["searchTerm0"], 0); var searchTerm1 = - parseInt(filter[key1][key2][key3].value["searchTerm1"]) / - 100; + safePercentageFilter(filter[key1][key2][key3].value["searchTerm1"], 1); item["searchTerm"] = [searchTerm0, searchTerm1]; } } From 48b67f2857cc384bd53cd10815c1b46a53a8d806 Mon Sep 17 00:00:00 2001 From: Domingos Date: Wed, 15 Feb 2023 11:22:15 -0300 Subject: [PATCH 06/33] Live Admin: m2m language group in modern country form now uses checkboxes. --- voyages/apps/past/admin.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/voyages/apps/past/admin.py b/voyages/apps/past/admin.py index 65952dbcd..ed0e34da6 100644 --- a/voyages/apps/past/admin.py +++ b/voyages/apps/past/admin.py @@ -1,6 +1,7 @@ from django.contrib import admin -from voyages.apps.voyage.forms import extract +from django import forms +from django.db import models from .models import LanguageGroup, AltLanguageGroupName, ModernCountry, RegisterCountry, CaptiveFate, CaptiveStatus @@ -17,8 +18,13 @@ class AltLanguageGroupNameInline(admin.TabularInline): class LanguageGroupForm(NamedModelFormAbstractBase): inlines = (AltLanguageGroupNameInline,) +class ModernCountryForm(NamedModelFormAbstractBase): + formfield_overrides = { + models.ManyToManyField: {'widget': forms.CheckboxSelectMultiple}, + } + admin.site.register(LanguageGroup, LanguageGroupForm) -admin.site.register(ModernCountry, NamedModelFormAbstractBase) +admin.site.register(ModernCountry, ModernCountryForm) admin.site.register(RegisterCountry, NamedModelFormAbstractBase) admin.site.register(CaptiveFate, NamedModelFormAbstractBase) admin.site.register(CaptiveStatus, NamedModelFormAbstractBase) \ No newline at end of file From 18a40466fb9aca9e79f0775ddae1fdf25bcec593 Mon Sep 17 00:00:00 2001 From: Domingos Date: Wed, 15 Feb 2023 17:26:52 -0300 Subject: [PATCH 07/33] Redirecting Introductory Maps to blog tag page. --- voyages/apps/voyage/urls.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/voyages/apps/voyage/urls.py b/voyages/apps/voyage/urls.py index dc51820f8..0cea8020d 100644 --- a/voyages/apps/voyage/urls.py +++ b/voyages/apps/voyage/urls.py @@ -6,6 +6,11 @@ import voyages.apps.static_content.views import voyages.apps.voyage.search_views import voyages.apps.voyage.views +from django.urls import reverse +from django.http import HttpResponsePermanentRedirect + +def redirect_to_blog_maps(_): + return HttpResponsePermanentRedirect(reverse('blog:tag', args=['intro-maps'])) urlpatterns = [ @@ -19,8 +24,7 @@ url(r'^downloads', TemplateView.as_view(template_name='downloads.html'), name='downloads'), - url(r'^maps', - TemplateView.as_view(template_name='maps.html'), name='maps'), + url(r'^maps$', redirect_to_blog, name='maps'), url(r'^ship', TemplateView.as_view(template_name='ship.html'), name='ship'), url(r'^navire', TemplateView.as_view(template_name='navire.html'), name='navire'), From de07e5bacdf28e7ebb82dd0031773675e388532c Mon Sep 17 00:00:00 2001 From: Domingos Date: Wed, 15 Feb 2023 18:45:39 -0300 Subject: [PATCH 08/33] Fix refactoring that did not propagate correctly... --- voyages/apps/voyage/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/voyages/apps/voyage/urls.py b/voyages/apps/voyage/urls.py index 0cea8020d..c6a324439 100644 --- a/voyages/apps/voyage/urls.py +++ b/voyages/apps/voyage/urls.py @@ -24,7 +24,7 @@ def redirect_to_blog_maps(_): url(r'^downloads', TemplateView.as_view(template_name='downloads.html'), name='downloads'), - url(r'^maps$', redirect_to_blog, name='maps'), + url(r'^maps$', redirect_to_blog_maps, name='maps'), url(r'^ship', TemplateView.as_view(template_name='ship.html'), name='ship'), url(r'^navire', TemplateView.as_view(template_name='navire.html'), name='navire'), From 16eac1dee568f1a223c72f342c5d1cc4854ccf14 Mon Sep 17 00:00:00 2001 From: Domingos Date: Thu, 16 Feb 2023 09:32:09 -0300 Subject: [PATCH 09/33] Preparing backend to handle Enslaved/Enslaver searches using the same post format as the saved queries for voyages. The initial (simpler) format is still supported so that previously saved queries for Enslaved/Enslaver may keep working. --- voyages/apps/past/models.py | 33 +++++++++++++++++++++------------ voyages/apps/past/views.py | 19 +++++++++++++++++-- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/voyages/apps/past/models.py b/voyages/apps/past/models.py index f0c743b5f..3c286f0a6 100644 --- a/voyages/apps/past/models.py +++ b/voyages/apps/past/models.py @@ -967,6 +967,13 @@ def _year_range_conv(range): """ return [',,' + str(y) for y in range] +def single_val(source): + try: + if isinstance(source, list): + source = source[0] + except: + pass + return source class EnslavedSearch: """ @@ -1022,7 +1029,6 @@ def __init__(self, exact or fuzzy @param: gender The gender ('male' or 'female'). @param: age_range A pair (a, b) where a is the min and b is maximum age - @param: voyage_dataset A list of voyage datasets that restrict the search. @param: height_range A pair (a, b) where a is the min and b is maximum height @param: year_range A pair (a, b) where a is the min voyage year and b @@ -1043,13 +1049,14 @@ def __init__(self, 'columnName': 'NAME', 'direction': 'ASC or DESC' }. Note that if the search is fuzzy, then the fallback value of order_by is the ranking of the fuzzy search. + @param: voyage_dataset A list of voyage datasets that restrict the search. @param: skin_color a textual description for skin color (Racial Descriptor) @param: vessel_fate a list of fates for the associated voyage vessel. """ - self.enslaved_dataset = enslaved_dataset - self.searched_name = searched_name - self.exact_name_search = exact_name_search - self.gender = gender + self.enslaved_dataset = single_val(enslaved_dataset) + self.searched_name = single_val(searched_name) + self.exact_name_search = single_val(exact_name_search) + self.gender = single_val(gender) self.age_range = age_range self.height_range = height_range self.year_range = year_range @@ -1057,13 +1064,13 @@ def __init__(self, self.disembarkation_ports = disembarkation_ports self.post_disembark_location = post_disembark_location self.language_groups = language_groups - self.ship_name = ship_name + self.ship_name = single_val(ship_name) self.voyage_id = voyage_id self.enslaved_id = enslaved_id - self.source = source + self.source = single_val(source) self.order_by = order_by or [{'columnName': 'pk', 'direction': 'asc'}] self.voyage_dataset = voyage_dataset - self.skin_color = skin_color + self.skin_color = single_val(skin_color) self.vessel_fate = vessel_fate def get_order_for_field(self, field): @@ -1414,20 +1421,22 @@ def __init__(self, associated enslaved count between a and b. @param: roles a list of role pks that should be matched (an enslaver may appear in multiple roles). + @param: a list of Voyage datasets that should be used to match voyages + connected to the enslaver. @param: order_by An array of dicts { 'columnName': 'NAME', 'direction': 'ASC or DESC' }. Note that if the search is fuzzy, then the fallback value of order_by is the ranking of the fuzzy search. """ - self.searched_name = searched_name - self.exact_name_search = exact_name_search + self.searched_name = single_val(searched_name) + self.exact_name_search = single_val(exact_name_search) self.year_range = year_range self.embarkation_ports = embarkation_ports self.disembarkation_ports = disembarkation_ports self.departure_ports = departure_ports - self.ship_name = ship_name + self.ship_name = single_val(ship_name) self.voyage_id = voyage_id - self.source = source + self.source = single_val(source) self.enslaved_count = enslaved_count self.roles = roles self.voyage_datasets = voyage_datasets diff --git a/voyages/apps/past/views.py b/voyages/apps/past/views.py index 4ddfd18a1..7d106eaf3 100644 --- a/voyages/apps/past/views.py +++ b/voyages/apps/past/views.py @@ -263,6 +263,19 @@ def refresh_maps_cache(request): refresh_maps_cache(None) +def process_search_query_post(user_query): + # Initially we started with a simple approach that does not include the + # operators on variables and thus can be immediately passed to the + # EnslavedSearch constructor. However, that decision causes pain for the + # saved query feature since the UI should know which operator was used for + # the query in order to properly reproduce it. We are therefore handling + # both situations here by simply removing what we don't need at the backend + # and allowing the saved query to follow the same format as before. + if 'items' in user_query: + # This is the newer format with the operation encoded. + user_query = {item['varName']: item['searchTerm'] for item in user_query['items']} + return user_query + @require_POST @csrf_exempt def search_enslaved(request): @@ -271,7 +284,8 @@ def search_enslaved(request): # decoded from the JSON body as arguments to the EnslavedSearch # constructor. data = json.loads(request.body) - search = EnslavedSearch(**data['search_query']) + user_query = process_search_query_post(data['search_query']) + search = EnslavedSearch(**user_query) fields=data.get('fields',None) output_type = data.get('output', 'resultsTable') @@ -508,7 +522,8 @@ def adapter(page): @csrf_exempt def search_enslaver(request): data = json.loads(request.body) - search = EnslaverSearch(**data['search_query']) + user_query = process_search_query_post(data['search_query']) + search = EnslaverSearch(**user_query) fields = data.get('fields') if fields is None: fields = [ From 43d3091fed9fdf8f684b545196352ef23fe262ea Mon Sep 17 00:00:00 2001 From: Domingos Date: Thu, 16 Feb 2023 16:13:23 -0300 Subject: [PATCH 10/33] Sorting Enslavers by name now possible using cached properties. We introduce min_alias and max_alias as precomputed cached properties which are initialized with only alphabetical characters in the identities' aliases, replacing any blank entries by extreme values so that those appear last in their respective sort. --- .../migrations/0028_auto_20230216_1250.py | 25 ++++++++++ voyages/apps/past/models.py | 47 +++++++++++++++++-- .../sitemedia/scripts/vue/enslavers/search.js | 2 +- 3 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 voyages/apps/past/migrations/0028_auto_20230216_1250.py diff --git a/voyages/apps/past/migrations/0028_auto_20230216_1250.py b/voyages/apps/past/migrations/0028_auto_20230216_1250.py new file mode 100644 index 000000000..8e8900f0a --- /dev/null +++ b/voyages/apps/past/migrations/0028_auto_20230216_1250.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.17 on 2023-02-16 12:50 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('past', '0027_auto_20230214_1408'), + ] + + operations = [ + migrations.AddField( + model_name='enslavercachedproperties', + name='max_alias', + field=models.CharField(blank=True, db_index=True, max_length=255), + ), + migrations.AddField( + model_name='enslavercachedproperties', + name='min_alias', + field=models.CharField(blank=True, db_index=True, max_length=255), + ), + ] diff --git a/voyages/apps/past/models.py b/voyages/apps/past/models.py index 3c286f0a6..19a24dab7 100644 --- a/voyages/apps/past/models.py +++ b/voyages/apps/past/models.py @@ -366,6 +366,10 @@ class BitsAndFunc(Func): template = '(%(expressions)s)' +class Lower(Func): + function = 'LOWER' + + class PowerFunc(Func): arity = 2 function = 'POWER' @@ -373,6 +377,13 @@ class PowerFunc(Func): template = 'POWER(%(expressions)s)' +class RemoveNonAlphaChars(Func): + arity = 1 + function = 'REGEXP_REPLACE' + arg_joiner = ',' + template = r"""REGEXP_REPLACE(%(expressions)s, '[^[[:alpha:]]]', '')""" + + class EnslaverCachedProperties(models.Model): identity = models.OneToOneField(EnslaverIdentity, related_name='cached_properties', on_delete=models.CASCADE, primary_key=True) @@ -386,6 +397,10 @@ class EnslaverCachedProperties(models.Model): # The voyage datasets that contain voyages associated with the enslavers. We # encode the aggregation as a bitwise OR of the powers of two of dataset values. voyage_datasets = models.IntegerField(db_index=True, null=True) + # Store lexicographically smallest and largest aliases for each Enslaver. + # This can then be used to sort + min_alias = models.CharField(blank=True, db_index=True, max_length=255) + max_alias = models.CharField(blank=True, db_index=True, max_length=255) @staticmethod def compute(identities=None): @@ -394,11 +409,12 @@ def compute(identities=None): EnslaverCachedProperties table which can be used to search and sort enslavers based on aggregated values. """ - def apply_filter(q): - return q.filter(**{f"{identity_field}__in": identities}) if identities is not None else q + identity_field = 'enslaver_alias__identity__id' + + def apply_filter(q, matching_field=None): + return q.filter(**{f"{matching_field or identity_field}__in": identities}) if identities is not None else q helper = PropHelper() - identity_field = 'enslaver_alias__identity__id' voyage_year_field = 'voyage__voyage_dates__imp_arrival_at_port_of_dis' q_years = EnslaverVoyageConnection.objects \ .select_related(voyage_year_field) \ @@ -420,6 +436,20 @@ def get_year(s): helper.set(int(row[0]), 'min_year', get_year(row[1])) helper.set(int(row[0]), 'max_year', get_year(row[2])) + # Process the alias field so that sorting behaves decently even with + # some badly formatted entries we currently have (e.g. with question + # marks, commas, digits etc) + alias_field = RemoveNonAlphaChars(Lower(F('alias'))) + q_aliases = EnslaverAlias.objects \ + .values('identity_id') \ + .annotate(min_alias=Min(alias_field)) \ + .annotate(max_alias=Max(alias_field)) \ + .values_list('identity_id', 'min_alias', 'max_alias') + q_aliases = apply_filter(q_aliases, 'identity_id') + for row in q_aliases: + helper.set(int(row[0]), 'min_alias', row[1]) + helper.set(int(row[0]), 'max_alias', row[2]) + q_datasets = EnslaverVoyageConnection.objects \ .select_related('voyage__dataset') \ .values(identity_field) \ @@ -490,6 +520,14 @@ def get_year(s): props.transactions_amount = item.get('tot_amount', 0) props.first_year = item.get('min_year', None) props.last_year = item.get('max_year', None) + props.min_alias = item.get('min_alias', '') + props.max_alias = item.get('max_alias', '') + # Avoid blank entries messing up alias sorting with some hardcoded + # balues that should place these last in their respective sorts. + if props.min_alias == '': + props.min_alias = 'zzzzzzzzzz' + if props.max_alias == '': + props.max_alias = 'aaaaaaaaaa' props.voyage_datasets = item.get('datasets', 0) yield props @@ -1533,6 +1571,9 @@ def add_voyage_field(q, field, op, val): orm_orderby = [] break is_desc = x['direction'].lower() == 'desc' + if col_name == 'alias_list': + orm_orderby.append(f"{'-' if is_desc else ''}cached_properties__{'max' if is_desc else 'min'}_alias") + continue order_field = F(col_name) if is_desc: order_field = order_field.desc(nulls_last=True) diff --git a/voyages/sitemedia/scripts/vue/enslavers/search.js b/voyages/sitemedia/scripts/vue/enslavers/search.js index 385ee898b..0c17651b5 100644 --- a/voyages/sitemedia/scripts/vue/enslavers/search.js +++ b/voyages/sitemedia/scripts/vue/enslavers/search.js @@ -13,7 +13,7 @@ const contributeCol = { data: "id", header: "", name: "contribute", className: " var allColumns = [ // name - { data: "alias_list", category: 0, header: gettext("Full Name"), isImputed: false, orderable: false }, + { data: "alias_list", category: 0, header: gettext("Full Name"), isImputed: false, orderable: true }, { data: "ranking", category: 0, header: gettext("Search Ranking"), isImputed: false, isUserSearchBased: true, visible: false }, // voyages From 57e49e7c1a5598c1542f4a3b8d2a30c8e2f1b1be Mon Sep 17 00:00:00 2001 From: Domingos Date: Fri, 17 Feb 2023 17:02:38 -0300 Subject: [PATCH 11/33] Removing country name from AO editorial review summary panel. --- .../apps/contribute/templates/contribute/review_origins.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/voyages/apps/contribute/templates/contribute/review_origins.html b/voyages/apps/contribute/templates/contribute/review_origins.html index f601be725..c525e0cb3 100644 --- a/voyages/apps/contribute/templates/contribute/review_origins.html +++ b/voyages/apps/contribute/templates/contribute/review_origins.html @@ -53,7 +53,7 @@

Contributed Language Groups

{% trans "Multilingual" %}
  • - [[ cl.language_group_name ]] ([[ cl.modern_country_name ]]) + [[ cl.language_group_name ]]
From 930a6d10a248eba801cb7368b59a34d16abfe064 Mon Sep 17 00:00:00 2001 From: Domingos Date: Fri, 17 Feb 2023 17:55:57 -0300 Subject: [PATCH 12/33] Separating enslaver variables in a new category "Enslaver Details". --- .../past-enslavers/_enslaver-details.html | 27 ++++++++++++ .../templates/past-enslavers/_itinerary.html | 11 +---- .../apps/past/templates/past/enslavers.html | 2 + .../sitemedia/scripts/vue/enslavers/app.js | 1 + .../sitemedia/scripts/vue/enslavers/search.js | 11 +++-- .../vue/variables/past-enslavers/details.js | 44 +++++++++++++++++++ .../vue/variables/past-enslavers/itinerary.js | 27 ------------ 7 files changed, 82 insertions(+), 41 deletions(-) create mode 100644 voyages/apps/past/templates/past-enslavers/_enslaver-details.html create mode 100644 voyages/sitemedia/scripts/vue/variables/past-enslavers/details.js diff --git a/voyages/apps/past/templates/past-enslavers/_enslaver-details.html b/voyages/apps/past/templates/past-enslavers/_enslaver-details.html new file mode 100644 index 000000000..1f1fa7612 --- /dev/null +++ b/voyages/apps/past/templates/past-enslavers/_enslaver-details.html @@ -0,0 +1,27 @@ +{% load i18n %} + + diff --git a/voyages/apps/past/templates/past-enslavers/_itinerary.html b/voyages/apps/past/templates/past-enslavers/_itinerary.html index f41caa052..c63f96cff 100644 --- a/voyages/apps/past/templates/past-enslavers/_itinerary.html +++ b/voyages/apps/past/templates/past-enslavers/_itinerary.html @@ -24,16 +24,7 @@ :filter="scope.filters.var_year_range" @change="changed"> - - - - - - - + diff --git a/voyages/apps/past/templates/past/enslavers.html b/voyages/apps/past/templates/past/enslavers.html index 39ddc56ee..d6f9b1592 100644 --- a/voyages/apps/past/templates/past/enslavers.html +++ b/voyages/apps/past/templates/past/enslavers.html @@ -44,6 +44,7 @@