From 15b6e7c81a7e5b2e1a5228d145c953dcd6413027 Mon Sep 17 00:00:00 2001 From: Derek Keller Date: Wed, 8 Feb 2023 09:20:31 -0600 Subject: [PATCH] Feature/importation (#183) * Initial planning and work on Voyages' enslaver migration. We are splitting the process in four steps to allow for ample testing and verification. In the intermediary steps, both legacy VoyageCaptain and VoyageOwner tables will be used alongside the Enslaver (PAST) data. A settings.py variable was created to ensure that we can advance or rollback on the steps. Once finalized, the plan is to drop the legacy tables. * Additional code to help with data migration Voyages to PAST. * Adding two extra intended disembarkation ports to VoyageItinerary. Guarding None values in _map_voyage_to_spss. Fixing bugs in export of AFRINFO / CARGO vars. * Disembarkation ports now selectable in OoK. * Fixing issues in export and import CSV for voyages. A roundtrip test: export to file 1, import from file 1, export to file 2 yielded identical files 1 and 2. * Fixing DataTable CSV/Excel export. Issue was new Linked Voyage column which contains array-valued cells. * pip and mysql library version bumps * removing moderncountry from enslaved language group contribution (as it is implicit in the lg) --------- Co-authored-by: Domingos Co-authored-by: John Mulligan --- .gitignore | 1 + docker/django/Dockerfile | 6 +- voyages/apps/common/export.py | 8 +- voyages/apps/common/utils.py | 15 +- voyages/apps/contribute/forms.py | 4 + .../migrations/0024_auto_20230131_1336.py | 27 + voyages/apps/contribute/models.py | 12 + voyages/apps/contribute/publication.py | 525 ++++++++++-------- .../templates/contribute/interim.html | 18 + .../templates/contribute/interim_summary.html | 14 + voyages/apps/contribute/tests.py | 4 + voyages/apps/contribute/views.py | 10 + voyages/apps/past/models.py | 67 +++ .../apps/past/templates/past/_itinerary.html | 1 - voyages/apps/past/views.py | 20 +- .../voyage/management/commands/importcsv.py | 163 +++++- .../migrations/0020_auto_20230131_1336.py | 36 ++ voyages/apps/voyage/models.py | 80 ++- voyages/apps/voyage/search_indexes.py | 14 +- voyages/apps/voyage/search_views.py | 1 + voyages/settings.py | 13 +- .../scripts/vue/past-contribute/app.js | 4 +- 22 files changed, 724 insertions(+), 319 deletions(-) create mode 100644 voyages/apps/contribute/migrations/0024_auto_20230131_1336.py create mode 100644 voyages/apps/voyage/migrations/0020_auto_20230131_1336.py diff --git a/.gitignore b/.gitignore index 9829f7368..8a5f488d9 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ google_auth.json documents/audio/** documents/blog/** documents/_versions/** +documents/csv_downloads/** diff --git a/docker/django/Dockerfile b/docker/django/Dockerfile index 3201f5790..002155eee 100644 --- a/docker/django/Dockerfile +++ b/docker/django/Dockerfile @@ -9,10 +9,10 @@ FROM ubuntu:18.04 AS base RUN apt-get update -y \ && apt-get install --yes --no-upgrade --no-install-recommends \ gettext=0.19.8.1-6ubuntu0.3 \ - mysql-client=5.7.40-0ubuntu0.18.04.1 \ - libmysqlclient-dev=5.7.40-0ubuntu0.18.04.1 \ + mysql-client=5.7.41-0ubuntu0.18.04.1 \ + libmysqlclient-dev=5.7.41-0ubuntu0.18.04.1 \ python3=3.6.7-1~18.04 \ - python3-pip=9.0.1-2.3~ubuntu1.18.04.5 \ + python3-pip=9.0.1-2.3~ubuntu1.18.04.6 \ python3-future=0.15.2-4ubuntu2 \ python3-pygit2=0.26.2-2 \ && apt-get clean \ diff --git a/voyages/apps/common/export.py b/voyages/apps/common/export.py index 781b8dc35..0c522c7a8 100644 --- a/voyages/apps/common/export.py +++ b/voyages/apps/common/export.py @@ -2,10 +2,13 @@ from builtins import str +import json import xlwt +from collections.abc import Iterable from django.http import HttpResponse + def download_xls(header_rows, data_set, row_header_columns=None): """ Generates an XLS file with the given data. @@ -52,7 +55,8 @@ def download_xls(header_rows, data_set, row_header_columns=None): # Write tabular data. for row in data_set: # TODO: use XLSX format that allows more rows! - if row_index == 65536: + if row_index == 65535: + ws.write(row_index, 0, 'Output is truncated!') break col_index = 0 for rhd in row_header_data: @@ -64,6 +68,8 @@ def download_xls(header_rows, data_set, row_header_columns=None): col_index) col_index += 1 for cell in row: + if not isinstance(cell, str) and isinstance(cell, Iterable): + cell = json.dumps(cell) ws.write(row_index, col_index, cell, number_style) col_index += 1 row_index += 1 diff --git a/voyages/apps/common/utils.py b/voyages/apps/common/utils.py index a7d425c1e..a030bb10c 100644 --- a/voyages/apps/common/utils.py +++ b/voyages/apps/common/utils.py @@ -8,17 +8,18 @@ empty = re.compile(r"^\s*\.?$") -def get_multi_valued_column_suffix(max_columns): - ALPHABET = 26 - if max_columns > 2 * ALPHABET: raise Exception("Too many columns!") - first_char = ord('a') - single_char_limit = min(max_columns, ALPHABET) +def get_multi_valued_column_suffix(max_columns, upper_case=False): + ALPHABET_LEN = 26 + if max_columns > 2 * ALPHABET_LEN: raise Exception("Too many columns!") + start_char = 'A' if upper_case else 'a' + first_char = ord(start_char) + single_char_limit = min(max_columns, ALPHABET_LEN) for i in range(0, single_char_limit): yield chr(first_char + i) - max_columns -= ALPHABET + max_columns -= ALPHABET_LEN if max_columns > 0: for i in range(0, max_columns): - yield 'a' + chr(first_char + i) + yield start_char + chr(first_char + i) class RowHelper: """ diff --git a/voyages/apps/contribute/forms.py b/voyages/apps/contribute/forms.py index 83cf7fb78..b56c23bf3 100644 --- a/voyages/apps/contribute/forms.py +++ b/voyages/apps/contribute/forms.py @@ -210,6 +210,10 @@ class Meta: _('First port of intended disembarkation'), 'second_port_intended_disembarkation': _('Second port of intended disembarkation'), + 'third_port_intended_disembarkation': + _('Third port of intended disembarkation'), + 'fourth_port_intended_disembarkation': + _('Fourth port of intended disembarkation'), 'port_of_departure': _('Port of departure'), 'number_of_ports_called_prior_to_slave_purchase': _('Number of ports called prior to slave purchase'), diff --git a/voyages/apps/contribute/migrations/0024_auto_20230131_1336.py b/voyages/apps/contribute/migrations/0024_auto_20230131_1336.py new file mode 100644 index 000000000..2fc31db03 --- /dev/null +++ b/voyages/apps/contribute/migrations/0024_auto_20230131_1336.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.17 on 2023-01-31 13:36 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('voyage', '0020_auto_20230131_1336'), + ('contribute', '0023_auto_20220621_1812'), + ] + + operations = [ + migrations.AddField( + model_name='interimvoyage', + name='fourth_port_intended_disembarkation', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='voyage.Place'), + ), + migrations.AddField( + model_name='interimvoyage', + name='third_port_intended_disembarkation', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='voyage.Place'), + ), + ] diff --git a/voyages/apps/contribute/models.py b/voyages/apps/contribute/models.py index db510d84c..e0d8127ac 100644 --- a/voyages/apps/contribute/models.py +++ b/voyages/apps/contribute/models.py @@ -127,6 +127,18 @@ class InterimVoyage(models.Model): null=True, blank=True, on_delete=models.CASCADE) + third_port_intended_disembarkation = models.ForeignKey( + voyage.models.Place, + related_name='+', + null=True, + blank=True, + on_delete=models.CASCADE) + fourth_port_intended_disembarkation = models.ForeignKey( + voyage.models.Place, + related_name='+', + null=True, + blank=True, + on_delete=models.CASCADE) port_of_departure = models.ForeignKey( voyage.models.Place, related_name='+', diff --git a/voyages/apps/contribute/publication.py b/voyages/apps/contribute/publication.py index 082ce78cd..3717ced0d 100644 --- a/voyages/apps/contribute/publication.py +++ b/voyages/apps/contribute/publication.py @@ -23,6 +23,7 @@ NewVoyageContribution, ReviewRequest, ReviewRequestDecision) +from voyages.apps.past.models import VoyageCaptainOwnerHelper from voyages.apps.voyage.models import (Voyage, VoyageCaptain, VoyageCaptainConnection, VoyageCargoConnection, VoyageCrew, VoyageDataset, VoyageDates, @@ -38,7 +39,7 @@ _exported_spss_fields = [ 'VOYAGEID', 'STATUS', 'ADLT1IMP', 'ADLT2IMP', 'ADLT3IMP', 'ADPSALE1', 'ADPSALE2', 'ADULT1', 'ADULT2', 'ADULT3', 'ADULT4', 'ADULT5', 'ADULT6', - 'ADULT7', 'ARRPORT', 'ARRPORT2', 'BOY1', 'BOY2', 'BOY3', 'BOY4', 'BOY5', + 'ADULT7', 'ARRPORT', 'ARRPORT2', 'ARRPORT3', 'ARRPORT4', 'BOY1', 'BOY2', 'BOY3', 'BOY4', 'BOY5', 'BOY6', 'BOY7', 'BOYRAT7', 'CAPTAINA', 'CAPTAINB', 'CAPTAINC', 'CHIL1IMP', 'CHIL2IMP', 'CHIL3IMP', 'CHILD1', 'CHILD2', 'CHILD3', 'CHILD4', 'CHILD5', 'CHILD6', 'CHILD7', 'CHILRAT7', 'CONSTREG', 'CREW', 'CREW1', 'CREW2', @@ -62,7 +63,7 @@ 'OWNERC', 'OWNERD', 'OWNERE', 'OWNERF', 'OWNERG', 'OWNERH', 'OWNERI', 'OWNERJ', 'OWNERK', 'OWNERL', 'OWNERM', 'OWNERN', 'OWNERO', 'OWNERP', 'PLAC1TRA', 'PLAC2TRA', 'PLAC3TRA', 'PLACCONS', 'PLACREG', 'PORTDEP', - 'PORTRET', 'PTDEPIMP', 'REGARR', 'REGARR2', 'REGDIS1', 'REGDIS2', + 'PORTRET', 'PTDEPIMP', 'REGARR', 'REGARR2', 'REGARR3', 'REGARR4', 'REGDIS1', 'REGDIS2', 'REGDIS3', 'REGEM1', 'REGEM2', 'REGEM3', 'REGISREG', 'RESISTANCE', 'RETRNREG', 'RETRNREG1', 'RIG', 'SAILD1', 'SAILD2', 'SAILD3', 'SAILD4', 'SAILD5', 'SHIPNAME', 'SLA1PORT', 'SLAARRIV', 'SLADAFRI', 'SLADAMER', @@ -80,10 +81,10 @@ "WOMRAT1", "BOYRAT3", "CHILRAT3", "GIRLRAT3", "MALRAT3", "MENRAT3", "WOMRAT3", "COMMENTS" ] + \ -["CARGOTYPE" + suffix.upper() for suffix in get_multi_valued_column_suffix(CARGO_COLUMN_COUNT)] + \ -["CARGOUNIT" + suffix.upper() for suffix in get_multi_valued_column_suffix(CARGO_COLUMN_COUNT)] + \ -["CARGOAMOUNT" + suffix.upper() for suffix in get_multi_valued_column_suffix(CARGO_COLUMN_COUNT)] + \ -["AFRINFO" + suffix.upper() for suffix in get_multi_valued_column_suffix(3)] +["CARGOTYPE" + suffix for suffix in get_multi_valued_column_suffix(CARGO_COLUMN_COUNT, True)] + \ +["CARGOUNIT" + suffix for suffix in get_multi_valued_column_suffix(CARGO_COLUMN_COUNT, True)] + \ +["CARGOAMOUNT" + suffix for suffix in get_multi_valued_column_suffix(CARGO_COLUMN_COUNT, True)] + \ +["AFRINFO" + suffix for suffix in get_multi_valued_column_suffix(3, True)] # TODO: Some variables are not an exact match to any field in or models, # so they either have some correspondence with a computed value from those @@ -396,6 +397,7 @@ def _get_label_value(x): def _get_region_value(place): return place.region.value if place else None +_captain_owner_helper = VoyageCaptainOwnerHelper() def _map_voyage_to_spss(voyage): data = {'STATUS': 'PUBLISHED'} @@ -403,34 +405,37 @@ def _map_voyage_to_spss(voyage): data['DATASET'] = voyage.dataset # Dates + def sanitize_date(csv_date): + return csv_date if csv_date != ',,' else '' + dates = voyage.voyage_dates - data['DATEDEP'] = dates.voyage_began - data['DATEEND'] = dates.voyage_completed - data['DATEBUY'] = dates.slave_purchase_began - data['DATELEFTAFR'] = dates.date_departed_africa - data['DATELAND1'] = dates.first_dis_of_slaves - data['DATELAND2'] = dates.arrival_at_second_place_landing - data['DATELAND3'] = dates.third_dis_of_slaves - data['DATEDEPAM'] = dates.departure_last_place_of_landing - _map_csv_date(data, 'DATEDEP', dates.voyage_began) - _map_csv_date(data, 'D1SLATR', dates.slave_purchase_began) - _map_csv_date(data, 'DLSLATR', dates.vessel_left_port) - _map_csv_date(data, 'DATARR3', dates.first_dis_of_slaves, '234') + data['DATEDEP'] = sanitize_date(dates.voyage_began) if dates else None + data['DATEEND'] = sanitize_date(dates.voyage_completed) if dates else None + data['DATEBUY'] = sanitize_date(dates.slave_purchase_began) if dates else None + data['DATELEFTAFR'] = sanitize_date(dates.date_departed_africa) if dates else None + data['DATELAND1'] = sanitize_date(dates.first_dis_of_slaves) if dates else None + data['DATELAND2'] = sanitize_date(dates.arrival_at_second_place_landing) if dates else None + data['DATELAND3'] = sanitize_date(dates.third_dis_of_slaves) if dates else None + data['DATEDEPAM'] = sanitize_date(dates.departure_last_place_of_landing) if dates else None + _map_csv_date(data, 'DATEDEP', dates.voyage_began if dates else None) + _map_csv_date(data, 'D1SLATR', dates.slave_purchase_began if dates else None) + _map_csv_date(data, 'DLSLATR', dates.vessel_left_port if dates else None) + _map_csv_date(data, 'DATARR3', dates.first_dis_of_slaves if dates else None, '234') _map_csv_date(data, 'DATARR3', - dates.arrival_at_second_place_landing, '678') - _map_csv_date(data, 'DATARR', dates.third_dis_of_slaves, + dates.arrival_at_second_place_landing if dates else None, '678') + _map_csv_date(data, 'DATARR', dates.third_dis_of_slaves if dates else None, ['39', '40', '41']) - _map_csv_date(data, 'DDEPAM', dates.departure_last_place_of_landing, + _map_csv_date(data, 'DDEPAM', dates.departure_last_place_of_landing if dates else None, ['', 'B', 'C']) - _map_csv_date(data, 'DATARR4', dates.voyage_completed, '345') + _map_csv_date(data, 'DATARR4', dates.voyage_completed if dates else None, '345') data['EVGREEN'] = voyage.voyage_in_cd_rom - data['VOYAGE'] = dates.length_middle_passage_days - data['YEARDEP'] = VoyageDates.get_date_year(dates.imp_voyage_began) - data['YEARAF'] = VoyageDates.get_date_year(dates.imp_departed_africa) - yearam = VoyageDates.get_date_year(dates.imp_arrival_at_port_of_dis) + data['VOYAGE'] = dates.length_middle_passage_days if dates else None + data['YEARDEP'] = VoyageDates.get_date_year(dates.imp_voyage_began if dates else None) + data['YEARAF'] = VoyageDates.get_date_year(dates.imp_departed_africa if dates else None) + yearam = VoyageDates.get_date_year(dates.imp_arrival_at_port_of_dis if dates else None) data['YEARAM'] = yearam - data['VOY1IMP'] = dates.imp_length_home_to_disembark - data['VOY2IMP'] = dates.imp_length_leaving_africa_to_disembark + data['VOY1IMP'] = dates.imp_length_home_to_disembark if dates else None + data['VOY2IMP'] = dates.imp_length_leaving_africa_to_disembark if dates else None data['YEAR5'] = year_mod(yearam, 5, 1500) data['YEAR10'] = year_mod(yearam, 10, 1500) data['YEAR25'] = year_mod(yearam, 25, 1500) @@ -451,245 +456,253 @@ def _map_voyage_to_spss(voyage): # Ship ship = voyage.voyage_ship - data['SHIPNAME'] = ship.ship_name - data['NATIONAL'] = _get_label_value(ship.nationality_ship) - data['TONNAGE'] = ship.tonnage - data['TONTYPE'] = _get_label_value(ship.ton_type) - data['RIG'] = _get_label_value(ship.rig_of_vessel) - data['GUNS'] = ship.guns_mounted - data['YRCONS'] = ship.year_of_construction - data['PLACCONS'] = _get_label_value(ship.vessel_construction_place) - data['CONSTREG'] = _get_label_value(ship.vessel_construction_region) - data['YRREG'] = ship.registered_year - data['PLACREG'] = _get_label_value(ship.registered_place) - data['REGISREG'] = _get_label_value(ship.registered_region) - - aux = list(get_multi_valued_column_suffix(16)) - for i, owner in enumerate(voyage.voyage_ship_owner.all()): + data['SHIPNAME'] = ship.ship_name if ship else None + data['NATIONAL'] = _get_label_value(ship.nationality_ship if ship else None) + data['TONNAGE'] = ship.tonnage if ship else None + data['TONTYPE'] = _get_label_value(ship.ton_type if ship else None) + data['RIG'] = _get_label_value(ship.rig_of_vessel if ship else None) + data['GUNS'] = ship.guns_mounted if ship else None + data['YRCONS'] = ship.year_of_construction if ship else None + data['PLACCONS'] = _get_label_value(ship.vessel_construction_place if ship else None) + data['CONSTREG'] = _get_label_value(ship.vessel_construction_region if ship else None) + data['YRREG'] = ship.registered_year if ship else None + data['PLACREG'] = _get_label_value(ship.registered_place if ship else None) + data['REGISREG'] = _get_label_value(ship.registered_region if ship else None) + data['NATINIMP'] = _get_label_value(ship.imputed_nationality if ship else None) + data['TONMOD'] = ship.tonnage_mod if ship else None + + aux = list(get_multi_valued_column_suffix(16, True)) + all_owners = _captain_owner_helper.get_owners(voyage) + for i, owner in enumerate(all_owners): if i >= len(aux): break - data['OWNER' + aux[i]] = owner.name - data['NATINIMP'] = _get_label_value(ship.imputed_nationality) - data['TONMOD'] = ship.tonnage_mod + data['OWNER' + aux[i]] = owner - aux = list(get_multi_valued_column_suffix(3)) - for i, captain in enumerate(voyage.voyage_captain.all()): + aux = list(get_multi_valued_column_suffix(3, True)) + all_captains = _captain_owner_helper.get_captains(voyage) + for i, captain in enumerate(all_captains): if i >= len(aux): break - data['CAPTAIN' + aux[i]] = captain.name + data['CAPTAIN' + aux[i]] = captain # Cargo - aux = list(get_multi_valued_column_suffix(CARGO_COLUMN_COUNT)) + aux = list(get_multi_valued_column_suffix(CARGO_COLUMN_COUNT, True)) for i, cargo_conn in enumerate(voyage.cargo.all()): if i >= len(aux): break - data['CARGOTYPE' + aux[i]] = cargo_conn.cargo.value - data['CARGOUNIT' + aux[i]] = cargo_conn.unit.value if cargo_conn.unit else None + data['CARGOTYPE' + aux[i]] = cargo_conn.cargo_id + data['CARGOUNIT' + aux[i]] = cargo_conn.unit_id data['CARGOAMOUNT' + aux[i]] = cargo_conn.amount # African info - aux = list(get_multi_valued_column_suffix(3)) + aux = list(get_multi_valued_column_suffix(3, True)) for i, afrinfo in enumerate(voyage.african_info.all()): if i >= len(aux): break - data['AFRINFO' + aux[i]] = afrinfo.value + data['AFRINFO' + aux[i]] = afrinfo.id # Itinerary itinerary = voyage.voyage_itinerary - data['PORTDEP'] = _get_label_value(itinerary.port_of_departure) - data['EMBPORT'] = _get_label_value(itinerary.int_first_port_emb) - data['EMBPORT2'] = _get_label_value(itinerary.int_second_port_emb) + data['PORTDEP'] = _get_label_value(itinerary.port_of_departure if itinerary else None) + data['EMBPORT'] = _get_label_value(itinerary.int_first_port_emb if itinerary else None) + data['EMBPORT2'] = _get_label_value(itinerary.int_second_port_emb if itinerary else None) data['EMBREG'] = _get_label_value( - itinerary.int_first_region_purchase_slaves) + itinerary.int_first_region_purchase_slaves if itinerary else None) data['EMBREG2'] = _get_label_value( - itinerary.int_second_region_purchase_slaves) - data['ARRPORT'] = _get_label_value(itinerary.int_first_port_dis) - data['ARRPORT2'] = _get_label_value(itinerary.int_second_port_dis) - data['REGARR'] = _get_label_value(itinerary.int_first_region_slave_landing) + itinerary.int_second_region_purchase_slaves if itinerary else None) + data['ARRPORT'] = _get_label_value(itinerary.int_first_port_dis if itinerary else None) + data['ARRPORT2'] = _get_label_value(itinerary.int_second_port_dis if itinerary else None) + data['ARRPORT3'] = _get_label_value(itinerary.int_third_port_dis if itinerary else None) + data['ARRPORT4'] = _get_label_value(itinerary.int_fourth_port_dis if itinerary else None) + data['REGARR'] = _get_label_value(itinerary.int_first_region_slave_landing if itinerary else None) data['REGARR2'] = _get_label_value( - itinerary.int_second_place_region_slave_landing) - data['NPPRETRA'] = itinerary.ports_called_buying_slaves - data['PLAC1TRA'] = _get_label_value(itinerary.first_place_slave_purchase) - data['PLAC2TRA'] = _get_label_value(itinerary.second_place_slave_purchase) - data['PLAC3TRA'] = _get_label_value(itinerary.third_place_slave_purchase) - data['REGEM1'] = _get_label_value(itinerary.first_region_slave_emb) - data['REGEM2'] = _get_label_value(itinerary.second_region_slave_emb) - data['REGEM3'] = _get_label_value(itinerary.third_region_slave_emb) + itinerary.int_second_place_region_slave_landing if itinerary else None) + data['REGARR3'] = _get_label_value( + itinerary.int_third_place_region_slave_landing if itinerary else None) + data['REGARR4'] = _get_label_value( + itinerary.int_fourth_place_region_slave_landing if itinerary else None) + data['NPPRETRA'] = itinerary.ports_called_buying_slaves if itinerary else None + data['PLAC1TRA'] = _get_label_value(itinerary.first_place_slave_purchase if itinerary else None) + data['PLAC2TRA'] = _get_label_value(itinerary.second_place_slave_purchase if itinerary else None) + data['PLAC3TRA'] = _get_label_value(itinerary.third_place_slave_purchase if itinerary else None) + data['REGEM1'] = _get_label_value(itinerary.first_region_slave_emb if itinerary else None) + data['REGEM2'] = _get_label_value(itinerary.second_region_slave_emb if itinerary else None) + data['REGEM3'] = _get_label_value(itinerary.third_region_slave_emb if itinerary else None) data['NPAFTTRA'] = _get_label_value( - itinerary.port_of_call_before_atl_crossing) - data['NPPRIOR'] = itinerary.number_of_ports_of_call - data['SLA1PORT'] = _get_label_value(itinerary.first_landing_place) - data['ADPSALE1'] = _get_label_value(itinerary.second_landing_place) - data['ADPSALE2'] = _get_label_value(itinerary.third_landing_place) - data['REGDIS1'] = _get_label_value(itinerary.first_landing_region) - data['REGDIS2'] = _get_label_value(itinerary.second_landing_region) - data['REGDIS3'] = _get_label_value(itinerary.third_landing_region) - data['PORTRET'] = _get_label_value(itinerary.place_voyage_ended) - data['RETRNREG'] = _get_label_value(itinerary.region_of_return) - data['RETRNREG1'] = _get_label_value(itinerary.broad_region_of_return) + itinerary.port_of_call_before_atl_crossing if itinerary else None) + data['NPPRIOR'] = itinerary.number_of_ports_of_call if itinerary else None + data['SLA1PORT'] = _get_label_value(itinerary.first_landing_place if itinerary else None) + data['ADPSALE1'] = _get_label_value(itinerary.second_landing_place if itinerary else None) + data['ADPSALE2'] = _get_label_value(itinerary.third_landing_place if itinerary else None) + data['REGDIS1'] = _get_label_value(itinerary.first_landing_region if itinerary else None) + data['REGDIS2'] = _get_label_value(itinerary.second_landing_region if itinerary else None) + data['REGDIS3'] = _get_label_value(itinerary.third_landing_region if itinerary else None) + data['PORTRET'] = _get_label_value(itinerary.place_voyage_ended if itinerary else None) + data['RETRNREG'] = _get_label_value(itinerary.region_of_return if itinerary else None) + data['RETRNREG1'] = _get_label_value(itinerary.broad_region_of_return if itinerary else None) data['MAJBUYPT'] = _get_label_value( - itinerary.principal_place_of_slave_purchase) - data['MAJSELPT'] = _get_label_value(itinerary.principal_port_of_slave_dis) - data['PTDEPIMP'] = _get_label_value(itinerary.imp_port_voyage_begin) + itinerary.principal_place_of_slave_purchase if itinerary else None) + data['MAJSELPT'] = _get_label_value(itinerary.principal_port_of_slave_dis if itinerary else None) + data['PTDEPIMP'] = _get_label_value(itinerary.imp_port_voyage_begin if itinerary else None) data['MJBYPTIMP'] = _get_label_value( - itinerary.imp_principal_place_of_slave_purchase) + itinerary.imp_principal_place_of_slave_purchase if itinerary else None) data['MAJBYIMP'] = _get_label_value( - itinerary.imp_principal_region_of_slave_purchase) + itinerary.imp_principal_region_of_slave_purchase if itinerary else None) data['MAJBYIMP1'] = _get_label_value( - itinerary.imp_broad_region_of_slave_purchase) + itinerary.imp_broad_region_of_slave_purchase if itinerary else None) data['MJSLPTIMP'] = _get_label_value( - itinerary.imp_principal_port_slave_dis) + itinerary.imp_principal_port_slave_dis if itinerary else None) data['MJSELIMP'] = _get_label_value( - itinerary.imp_principal_region_slave_dis) - data['MJSELIMP1'] = _get_label_value(itinerary.imp_broad_region_slave_dis) - data['DEPTREGIMP'] = _get_label_value(itinerary.imp_region_voyage_begin) + itinerary.imp_principal_region_slave_dis if itinerary else None) + data['MJSELIMP1'] = _get_label_value(itinerary.imp_broad_region_slave_dis if itinerary else None) + data['DEPTREGIMP'] = _get_label_value(itinerary.imp_region_voyage_begin if itinerary else None) data['DEPTREGIMP1'] = _get_label_value( - itinerary.imp_broad_region_voyage_begin) + itinerary.imp_broad_region_voyage_begin if itinerary else None) # Crew crew = voyage.voyage_crew - data['CREW1'] = crew.crew_voyage_outset - data['CREW2'] = crew.crew_departure_last_port - data['CREW3'] = crew.crew_first_landing - data['CREW4'] = crew.crew_return_begin - data['CREW5'] = crew.crew_end_voyage - data['CREW'] = crew.unspecified_crew - data['SAILD1'] = crew.crew_died_before_first_trade - data['SAILD2'] = crew.crew_died_while_ship_african - data['SAILD3'] = crew.crew_died_middle_passage - data['SAILD4'] = crew.crew_died_in_americas - data['SAILD5'] = crew.crew_died_on_return_voyage - data['CREWDIED'] = crew.crew_died_complete_voyage - data['NDESERT'] = crew.crew_deserted + data['CREW1'] = crew.crew_voyage_outset if crew else None + data['CREW2'] = crew.crew_departure_last_port if crew else None + data['CREW3'] = crew.crew_first_landing if crew else None + data['CREW4'] = crew.crew_return_begin if crew else None + data['CREW5'] = crew.crew_end_voyage if crew else None + data['CREW'] = crew.unspecified_crew if crew else None + data['SAILD1'] = crew.crew_died_before_first_trade if crew else None + data['SAILD2'] = crew.crew_died_while_ship_african if crew else None + data['SAILD3'] = crew.crew_died_middle_passage if crew else None + data['SAILD4'] = crew.crew_died_in_americas if crew else None + data['SAILD5'] = crew.crew_died_on_return_voyage if crew else None + data['CREWDIED'] = crew.crew_died_complete_voyage if crew else None + data['NDESERT'] = crew.crew_deserted if crew else None # Numbers numbers = voyage.voyage_slaves_numbers - data['SLADAFRI'] = numbers.slave_deaths_before_africa - data['SLADVOY'] = numbers.slave_deaths_between_africa_america - data['SLADAMER'] = numbers.slave_deaths_between_arrival_and_sale - data['SLINTEND'] = numbers.num_slaves_intended_first_port - data['SLINTEN2'] = numbers.num_slaves_intended_second_port - data['NCAR13'] = numbers.num_slaves_carried_first_port - data['NCAR15'] = numbers.num_slaves_carried_second_port - data['NCAR17'] = numbers.num_slaves_carried_third_port - data['TSLAVESP'] = numbers.total_num_slaves_purchased - data['TSLAVESD'] = numbers.total_num_slaves_dep_last_slaving_port - data['SLAARRIV'] = numbers.total_num_slaves_arr_first_port_embark - data['SLAS32'] = numbers.num_slaves_disembark_first_place - data['SLAS36'] = numbers.num_slaves_disembark_second_place - data['SLAS39'] = numbers.num_slaves_disembark_third_place - data['SLAXIMP'] = numbers.imp_total_num_slaves_embarked - data['SLAMIMP'] = numbers.imp_total_num_slaves_disembarked - data['JAMCASPR'] = numbers.imp_jamaican_cash_price - data['VYMRTIMP'] = numbers.imp_mortality_during_voyage - data['MEN1'] = numbers.num_men_embark_first_port_purchase - data['WOMEN1'] = numbers.num_women_embark_first_port_purchase - data['BOY1'] = numbers.num_boy_embark_first_port_purchase - data['GIRL1'] = numbers.num_girl_embark_first_port_purchase - data['ADULT1'] = numbers.num_adult_embark_first_port_purchase - data['CHILD1'] = numbers.num_child_embark_first_port_purchase - data['INFANT1'] = numbers.num_infant_embark_first_port_purchase - data['MALE1'] = numbers.num_males_embark_first_port_purchase - data['FEMALE1'] = numbers.num_females_embark_first_port_purchase - data['MEN2'] = numbers.num_men_died_middle_passage - data['WOMEN2'] = numbers.num_women_died_middle_passage - data['BOY2'] = numbers.num_boy_died_middle_passage - data['GIRL2'] = numbers.num_girl_died_middle_passage - data['ADULT2'] = numbers.num_adult_died_middle_passage - data['CHILD2'] = numbers.num_child_died_middle_passage - data['INFANT2'] = numbers.num_infant_died_middle_passage - data['MALE2'] = numbers.num_males_died_middle_passage - data['FEMALE2'] = numbers.num_females_died_middle_passage - data['MEN3'] = numbers.num_men_disembark_first_landing - data['WOMEN3'] = numbers.num_women_disembark_first_landing - data['BOY3'] = numbers.num_boy_disembark_first_landing - data['GIRL3'] = numbers.num_girl_disembark_first_landing - data['ADULT3'] = numbers.num_adult_disembark_first_landing - data['CHILD3'] = numbers.num_child_disembark_first_landing - data['INFANT3'] = numbers.num_infant_disembark_first_landing - data['MALE3'] = numbers.num_males_disembark_first_landing - data['FEMALE3'] = numbers.num_females_disembark_first_landing - data['MEN4'] = numbers.num_men_embark_second_port_purchase - data['WOMEN4'] = numbers.num_women_embark_second_port_purchase - data['BOY4'] = numbers.num_boy_embark_second_port_purchase - data['GIRL4'] = numbers.num_girl_embark_second_port_purchase - data['ADULT4'] = numbers.num_adult_embark_second_port_purchase - data['CHILD4'] = numbers.num_child_embark_second_port_purchase - data['INFANT4'] = numbers.num_infant_embark_second_port_purchase - data['MALE4'] = numbers.num_males_embark_second_port_purchase - data['FEMALE4'] = numbers.num_females_embark_second_port_purchase - data['MEN5'] = numbers.num_men_embark_third_port_purchase - data['WOMEN5'] = numbers.num_women_embark_third_port_purchase - data['BOY5'] = numbers.num_boy_embark_third_port_purchase - data['GIRL5'] = numbers.num_girl_embark_third_port_purchase - data['ADULT5'] = numbers.num_adult_embark_third_port_purchase - data['CHILD5'] = numbers.num_child_embark_third_port_purchase - data['INFANT5'] = numbers.num_infant_embark_third_port_purchase - data['MALE5'] = numbers.num_males_embark_third_port_purchase - data['FEMALE5'] = numbers.num_females_embark_third_port_purchase - data['MEN6'] = numbers.num_men_disembark_second_landing - data['WOMEN6'] = numbers.num_women_disembark_second_landing - data['BOY6'] = numbers.num_boy_disembark_second_landing - data['GIRL6'] = numbers.num_girl_disembark_second_landing - data['ADULT6'] = numbers.num_adult_disembark_second_landing - data['CHILD6'] = numbers.num_child_disembark_second_landing - data['INFANT6'] = numbers.num_infant_disembark_second_landing - data['MALE6'] = numbers.num_males_disembark_second_landing - data['FEMALE6'] = numbers.num_females_disembark_second_landing - data['ADLT1IMP'] = numbers.imp_num_adult_embarked - data['CHIL1IMP'] = numbers.imp_num_children_embarked - data['MALE1IMP'] = numbers.imp_num_male_embarked - data['FEML1IMP'] = numbers.imp_num_female_embarked - data['SLAVEMA1'] = numbers.total_slaves_embarked_age_identified - data['SLAVEMX1'] = numbers.total_slaves_embarked_gender_identified - data['ADLT2IMP'] = numbers.imp_adult_death_middle_passage - data['CHIL2IMP'] = numbers.imp_child_death_middle_passage - data['MALE2IMP'] = numbers.imp_male_death_middle_passage - data['FEML2IMP'] = numbers.imp_female_death_middle_passage - data['ADLT3IMP'] = numbers.imp_num_adult_landed - data['CHIL3IMP'] = numbers.imp_num_child_landed - data['MALE3IMP'] = numbers.imp_num_male_landed - data['FEML3IMP'] = numbers.imp_num_female_landed - data['SLAVEMA3'] = numbers.total_slaves_landed_age_identified - data['SLAVEMX3'] = numbers.total_slaves_landed_gender_identified - data['SLAVEMA7'] = numbers.total_slaves_dept_or_arr_age_identified - data['SLAVEMX7'] = numbers.total_slaves_dept_or_arr_gender_identified - data['SLAVMAX1'] = numbers.total_slaves_embarked_age_gender_identified + data['SLADAFRI'] = numbers.slave_deaths_before_africa if numbers else None + data['SLADVOY'] = numbers.slave_deaths_between_africa_america if numbers else None + data['SLADAMER'] = numbers.slave_deaths_between_arrival_and_sale if numbers else None + data['SLINTEND'] = numbers.num_slaves_intended_first_port if numbers else None + data['SLINTEN2'] = numbers.num_slaves_intended_second_port if numbers else None + data['NCAR13'] = numbers.num_slaves_carried_first_port if numbers else None + data['NCAR15'] = numbers.num_slaves_carried_second_port if numbers else None + data['NCAR17'] = numbers.num_slaves_carried_third_port if numbers else None + data['TSLAVESP'] = numbers.total_num_slaves_purchased if numbers else None + data['TSLAVESD'] = numbers.total_num_slaves_dep_last_slaving_port if numbers else None + data['SLAARRIV'] = numbers.total_num_slaves_arr_first_port_embark if numbers else None + data['SLAS32'] = numbers.num_slaves_disembark_first_place if numbers else None + data['SLAS36'] = numbers.num_slaves_disembark_second_place if numbers else None + data['SLAS39'] = numbers.num_slaves_disembark_third_place if numbers else None + data['SLAXIMP'] = numbers.imp_total_num_slaves_embarked if numbers else None + data['SLAMIMP'] = numbers.imp_total_num_slaves_disembarked if numbers else None + data['JAMCASPR'] = numbers.imp_jamaican_cash_price if numbers else None + data['VYMRTIMP'] = numbers.imp_mortality_during_voyage if numbers else None + data['MEN1'] = numbers.num_men_embark_first_port_purchase if numbers else None + data['WOMEN1'] = numbers.num_women_embark_first_port_purchase if numbers else None + data['BOY1'] = numbers.num_boy_embark_first_port_purchase if numbers else None + data['GIRL1'] = numbers.num_girl_embark_first_port_purchase if numbers else None + data['ADULT1'] = numbers.num_adult_embark_first_port_purchase if numbers else None + data['CHILD1'] = numbers.num_child_embark_first_port_purchase if numbers else None + data['INFANT1'] = numbers.num_infant_embark_first_port_purchase if numbers else None + data['MALE1'] = numbers.num_males_embark_first_port_purchase if numbers else None + data['FEMALE1'] = numbers.num_females_embark_first_port_purchase if numbers else None + data['MEN2'] = numbers.num_men_died_middle_passage if numbers else None + data['WOMEN2'] = numbers.num_women_died_middle_passage if numbers else None + data['BOY2'] = numbers.num_boy_died_middle_passage if numbers else None + data['GIRL2'] = numbers.num_girl_died_middle_passage if numbers else None + data['ADULT2'] = numbers.num_adult_died_middle_passage if numbers else None + data['CHILD2'] = numbers.num_child_died_middle_passage if numbers else None + data['INFANT2'] = numbers.num_infant_died_middle_passage if numbers else None + data['MALE2'] = numbers.num_males_died_middle_passage if numbers else None + data['FEMALE2'] = numbers.num_females_died_middle_passage if numbers else None + data['MEN3'] = numbers.num_men_disembark_first_landing if numbers else None + data['WOMEN3'] = numbers.num_women_disembark_first_landing if numbers else None + data['BOY3'] = numbers.num_boy_disembark_first_landing if numbers else None + data['GIRL3'] = numbers.num_girl_disembark_first_landing if numbers else None + data['ADULT3'] = numbers.num_adult_disembark_first_landing if numbers else None + data['CHILD3'] = numbers.num_child_disembark_first_landing if numbers else None + data['INFANT3'] = numbers.num_infant_disembark_first_landing if numbers else None + data['MALE3'] = numbers.num_males_disembark_first_landing if numbers else None + data['FEMALE3'] = numbers.num_females_disembark_first_landing if numbers else None + data['MEN4'] = numbers.num_men_embark_second_port_purchase if numbers else None + data['WOMEN4'] = numbers.num_women_embark_second_port_purchase if numbers else None + data['BOY4'] = numbers.num_boy_embark_second_port_purchase if numbers else None + data['GIRL4'] = numbers.num_girl_embark_second_port_purchase if numbers else None + data['ADULT4'] = numbers.num_adult_embark_second_port_purchase if numbers else None + data['CHILD4'] = numbers.num_child_embark_second_port_purchase if numbers else None + data['INFANT4'] = numbers.num_infant_embark_second_port_purchase if numbers else None + data['MALE4'] = numbers.num_males_embark_second_port_purchase if numbers else None + data['FEMALE4'] = numbers.num_females_embark_second_port_purchase if numbers else None + data['MEN5'] = numbers.num_men_embark_third_port_purchase if numbers else None + data['WOMEN5'] = numbers.num_women_embark_third_port_purchase if numbers else None + data['BOY5'] = numbers.num_boy_embark_third_port_purchase if numbers else None + data['GIRL5'] = numbers.num_girl_embark_third_port_purchase if numbers else None + data['ADULT5'] = numbers.num_adult_embark_third_port_purchase if numbers else None + data['CHILD5'] = numbers.num_child_embark_third_port_purchase if numbers else None + data['INFANT5'] = numbers.num_infant_embark_third_port_purchase if numbers else None + data['MALE5'] = numbers.num_males_embark_third_port_purchase if numbers else None + data['FEMALE5'] = numbers.num_females_embark_third_port_purchase if numbers else None + data['MEN6'] = numbers.num_men_disembark_second_landing if numbers else None + data['WOMEN6'] = numbers.num_women_disembark_second_landing if numbers else None + data['BOY6'] = numbers.num_boy_disembark_second_landing if numbers else None + data['GIRL6'] = numbers.num_girl_disembark_second_landing if numbers else None + data['ADULT6'] = numbers.num_adult_disembark_second_landing if numbers else None + data['CHILD6'] = numbers.num_child_disembark_second_landing if numbers else None + data['INFANT6'] = numbers.num_infant_disembark_second_landing if numbers else None + data['MALE6'] = numbers.num_males_disembark_second_landing if numbers else None + data['FEMALE6'] = numbers.num_females_disembark_second_landing if numbers else None + data['ADLT1IMP'] = numbers.imp_num_adult_embarked if numbers else None + data['CHIL1IMP'] = numbers.imp_num_children_embarked if numbers else None + data['MALE1IMP'] = numbers.imp_num_male_embarked if numbers else None + data['FEML1IMP'] = numbers.imp_num_female_embarked if numbers else None + data['SLAVEMA1'] = numbers.total_slaves_embarked_age_identified if numbers else None + data['SLAVEMX1'] = numbers.total_slaves_embarked_gender_identified if numbers else None + data['ADLT2IMP'] = numbers.imp_adult_death_middle_passage if numbers else None + data['CHIL2IMP'] = numbers.imp_child_death_middle_passage if numbers else None + data['MALE2IMP'] = numbers.imp_male_death_middle_passage if numbers else None + data['FEML2IMP'] = numbers.imp_female_death_middle_passage if numbers else None + data['ADLT3IMP'] = numbers.imp_num_adult_landed if numbers else None + data['CHIL3IMP'] = numbers.imp_num_child_landed if numbers else None + data['MALE3IMP'] = numbers.imp_num_male_landed if numbers else None + data['FEML3IMP'] = numbers.imp_num_female_landed if numbers else None + data['SLAVEMA3'] = numbers.total_slaves_landed_age_identified if numbers else None + data['SLAVEMX3'] = numbers.total_slaves_landed_gender_identified if numbers else None + data['SLAVEMA7'] = numbers.total_slaves_dept_or_arr_age_identified if numbers else None + data['SLAVEMX7'] = numbers.total_slaves_dept_or_arr_gender_identified if numbers else None + data['SLAVMAX1'] = numbers.total_slaves_embarked_age_gender_identified if numbers else None data['SLAVMAX3'] = ( - numbers.total_slaves_by_age_gender_identified_among_landed) + numbers.total_slaves_by_age_gender_identified_among_landed if numbers else None) data['SLAVMAX7'] = ( - numbers.total_slaves_by_age_gender_identified_departure_or_arrival) - data['TSLMTIMP'] = numbers.imp_slaves_embarked_for_mortality - data['MEN7'] = numbers.imp_num_men_total - data['WOMEN7'] = numbers.imp_num_women_total - data['BOY7'] = numbers.imp_num_boy_total - data['GIRL7'] = numbers.imp_num_girl_total - data['ADULT7'] = numbers.imp_num_adult_total - data['CHILD7'] = numbers.imp_num_child_total - data['MALE7'] = numbers.imp_num_males_total - data['FEMALE7'] = numbers.imp_num_females_total - data['MENRAT7'] = numbers.percentage_men - data['WOMRAT7'] = numbers.percentage_women - data['BOYRAT7'] = numbers.percentage_boy - data['GIRLRAT7'] = numbers.percentage_girl - data['MALRAT7'] = numbers.percentage_male - data['CHILRAT7'] = numbers.percentage_child - data['VYMRTRAT'] = numbers.imp_mortality_ratio - data["BOYRAT1"] = numbers.percentage_boys_among_embarked_slaves - data["CHILRAT1"] = numbers.child_ratio_among_embarked_slaves - data["GIRLRAT1"] = numbers.percentage_girls_among_embarked_slaves - data["MALRAT1"] = numbers.male_ratio_among_embarked_slaves - data["MENRAT1"] = numbers.percentage_men_among_embarked_slaves - data["WOMRAT1"] = numbers.percentage_women_among_embarked_slaves - data["BOYRAT3"] = numbers.percentage_boys_among_landed_slaves - data["CHILRAT3"] = numbers.child_ratio_among_landed_slaves - data["GIRLRAT3"] = numbers.percentage_girls_among_landed_slaves - data["MALRAT3"] = numbers.male_ratio_among_landed_slaves - data["MENRAT3"] = numbers.percentage_men_among_landed_slaves - data["WOMRAT3"] = numbers.percentage_women_among_landed_slaves + numbers.total_slaves_by_age_gender_identified_departure_or_arrival if numbers else None) + data['TSLMTIMP'] = numbers.imp_slaves_embarked_for_mortality if numbers else None + data['MEN7'] = numbers.imp_num_men_total if numbers else None + data['WOMEN7'] = numbers.imp_num_women_total if numbers else None + data['BOY7'] = numbers.imp_num_boy_total if numbers else None + data['GIRL7'] = numbers.imp_num_girl_total if numbers else None + data['ADULT7'] = numbers.imp_num_adult_total if numbers else None + data['CHILD7'] = numbers.imp_num_child_total if numbers else None + data['MALE7'] = numbers.imp_num_males_total if numbers else None + data['FEMALE7'] = numbers.imp_num_females_total if numbers else None + data['MENRAT7'] = numbers.percentage_men if numbers else None + data['WOMRAT7'] = numbers.percentage_women if numbers else None + data['BOYRAT7'] = numbers.percentage_boy if numbers else None + data['GIRLRAT7'] = numbers.percentage_girl if numbers else None + data['MALRAT7'] = numbers.percentage_male if numbers else None + data['CHILRAT7'] = numbers.percentage_child if numbers else None + data['VYMRTRAT'] = numbers.imp_mortality_ratio if numbers else None + data["BOYRAT1"] = numbers.percentage_boys_among_embarked_slaves if numbers else None + data["CHILRAT1"] = numbers.child_ratio_among_embarked_slaves if numbers else None + data["GIRLRAT1"] = numbers.percentage_girls_among_embarked_slaves if numbers else None + data["MALRAT1"] = numbers.male_ratio_among_embarked_slaves if numbers else None + data["MENRAT1"] = numbers.percentage_men_among_embarked_slaves if numbers else None + data["WOMRAT1"] = numbers.percentage_women_among_embarked_slaves if numbers else None + data["BOYRAT3"] = numbers.percentage_boys_among_landed_slaves if numbers else None + data["CHILRAT3"] = numbers.child_ratio_among_landed_slaves if numbers else None + data["GIRLRAT3"] = numbers.percentage_girls_among_landed_slaves if numbers else None + data["MALRAT3"] = numbers.male_ratio_among_landed_slaves if numbers else None + data["MENRAT3"] = numbers.percentage_men_among_landed_slaves if numbers else None + data["WOMRAT3"] = numbers.percentage_women_among_landed_slaves if numbers else None # INSERT HERE any new number variables [export CSV] - aux = list(get_multi_valued_column_suffix(18)) + aux = list(get_multi_valued_column_suffix(18, True)) for i, source_conn in enumerate(voyage.group.all()): if i >= len(aux): break @@ -766,10 +779,18 @@ def _map_interim_to_spss(interim): interim.first_port_intended_disembarkation) data['ARRPORT2'] = _get_label_value( interim.second_port_intended_disembarkation) + data['ARRPORT3'] = _get_label_value( + interim.third_port_intended_disembarkation) + data['ARRPORT4'] = _get_label_value( + interim.fourth_port_intended_disembarkation) data['REGARR'] = _get_region_value( interim.first_port_intended_disembarkation) data['REGARR2'] = _get_region_value( interim.second_port_intended_disembarkation) + data['REGARR3'] = _get_region_value( + interim.third_port_intended_disembarkation) + data['REGARR4'] = _get_region_value( + interim.fourth_port_intended_disembarkation) data['NPPRETRA'] = interim.number_of_ports_called_prior_to_slave_purchase data['PLAC1TRA'] = _get_label_value(interim.first_place_of_slave_purchase) data['PLAC2TRA'] = _get_label_value(interim.second_place_of_slave_purchase) @@ -858,7 +879,7 @@ def _map_interim_to_spss(interim): interim.primary_sources.all())) source_refs = [x.source_ref_text for x in created_sources] + \ [x.full_ref for x in interim.pre_existing_sources.all()] - aux = list(get_multi_valued_column_suffix(18)) + aux = list(get_multi_valued_column_suffix(18, True)) for i, ref in enumerate(source_refs): if i >= len(aux): break @@ -949,14 +970,19 @@ def broad_region(place): # Voyage Ship Owners def create_ship_owner(owner_name, order): - owner = VoyageShipOwner() - owner.name = owner_name - owner.save() - conn = VoyageShipOwnerConnection() - conn.owner = owner - conn.owner_order = order - conn.voyage = voyage - conn.save() + if settings.VOYAGE_ENSLAVERS_MIGRATION_STAGE <= 2: + owner = VoyageShipOwner() + owner.name = owner_name + owner.save() + conn = VoyageShipOwnerConnection() + conn.owner = owner + conn.owner_order = order + conn.voyage = voyage + conn.save() + if settings.VOYAGE_ENSLAVERS_MIGRATION_STAGE <= 2: + # TODO detect existing alias/identity and create connection or + # create new identity/alias if needed. + pass if interim.first_ship_owner: create_ship_owner(interim.first_ship_owner, 1) @@ -968,14 +994,19 @@ def create_ship_owner(owner_name, order): # Voyage Ship Captains def create_captain(name, order): - captain = VoyageCaptain() - captain.name = name - captain.save() - conn = VoyageCaptainConnection() - conn.captain = captain - conn.captain_order = order - conn.voyage = voyage - conn.save() + if settings.VOYAGE_ENSLAVERS_MIGRATION_STAGE <= 2: + captain = VoyageCaptain() + captain.name = name + captain.save() + conn = VoyageCaptainConnection() + conn.captain = captain + conn.captain_order = order + conn.voyage = voyage + conn.save() + if settings.VOYAGE_ENSLAVERS_MIGRATION_STAGE <= 2: + # TODO detect existing alias/identity and create connection or + # create new identity/alias if needed. + pass if interim.first_captain: create_captain(interim.first_captain, 1) @@ -998,10 +1029,16 @@ def create_captain(name, order): interim.first_port_intended_disembarkation) itinerary.int_second_port_dis = ( interim.second_port_intended_disembarkation) + itinerary.int_third_port_dis = interim.third_port_intended_disembarkation + itinerary.int_fourth_port_dis = interim.fourth_port_intended_disembarkation itinerary.int_first_region_slave_landing = region( interim.first_port_intended_disembarkation) itinerary.int_second_place_region_slave_landing = region( interim.second_port_intended_disembarkation) + itinerary.int_third_place_region_slave_landing = region( + interim.third_port_intended_disembarkation) + itinerary.int_fourth_place_region_slave_landing = region( + interim.fourth_port_intended_disembarkation) itinerary.ports_called_buying_slaves = ( interim.number_of_ports_called_prior_to_slave_purchase) itinerary.first_place_slave_purchase = ( diff --git a/voyages/apps/contribute/templates/contribute/interim.html b/voyages/apps/contribute/templates/contribute/interim.html index 21e322143..35f3da019 100644 --- a/voyages/apps/contribute/templates/contribute/interim.html +++ b/voyages/apps/contribute/templates/contribute/interim.html @@ -441,6 +441,24 @@ {% if f.help_text %}

{{ f.help_text }}

{% endif %}{% endwith %} +
+ {% with form.third_port_intended_disembarkation as f %} + {{ f.errors }} + {{ f.label_tag }} +
+ +
+ {% if f.help_text %}

{{ f.help_text }}

{% endif %}{% endwith %} +
+
+ {% with form.fourth_port_intended_disembarkation as f %} + {{ f.errors }} + {{ f.label_tag }} +
+ +
+ {% if f.help_text %}

{{ f.help_text }}

{% endif %}{% endwith %} +
{% with form.port_of_departure as f %} {{ f.errors }} diff --git a/voyages/apps/contribute/templates/contribute/interim_summary.html b/voyages/apps/contribute/templates/contribute/interim_summary.html index 8d10c43fd..fed13826c 100644 --- a/voyages/apps/contribute/templates/contribute/interim_summary.html +++ b/voyages/apps/contribute/templates/contribute/interim_summary.html @@ -176,6 +176,20 @@

{% trans "Voyage itinerary" %}

{{ f|selected_choice|default_if_none:blank_text }}
{% endwith %} + {% with form.third_port_intended_disembarkation as f %} +
+ {{ f.label_tag }} +
+ {{ f|selected_choice|default_if_none:blank_text }} +
+ {% endwith %} + {% with form.fourth_port_intended_disembarkation as f %} +
+ {{ f.label_tag }} +
+ {{ f|selected_choice|default_if_none:blank_text }} +
+ {% endwith %} {% with form.number_of_ports_called_prior_to_slave_purchase as f %}
{{ f.label_tag }} diff --git a/voyages/apps/contribute/tests.py b/voyages/apps/contribute/tests.py index 5582c8add..1244d15fd 100644 --- a/voyages/apps/contribute/tests.py +++ b/voyages/apps/contribute/tests.py @@ -287,6 +287,10 @@ def date_from_triple(m, d, y): dikt['arrport']) interim.second_port_intended_disembarkation = place_from_value( dikt['arrport2']) + interim.third_port_intended_disembarkation = place_from_value( + dikt['arrport3']) + interim.fourth_port_intended_disembarkation = place_from_value( + dikt['arrport4']) interim.port_of_departure = place_from_value(dikt['portdep']) interim.number_of_ports_called_prior_to_slave_purchase = dikt[ 'nppretra'] diff --git a/voyages/apps/contribute/views.py b/voyages/apps/contribute/views.py index 3ea24430f..36836f01a 100644 --- a/voyages/apps/contribute/views.py +++ b/voyages/apps/contribute/views.py @@ -857,6 +857,12 @@ def get_label(obj, field='name'): dikt[ 'second_port_intended_disembarkation' ] = itin.int_second_port_dis_id + dikt[ + 'third_port_intended_disembarkation' + ] = itin.int_third_port_dis_id + dikt[ + 'fourth_port_intended_disembarkation' + ] = itin.int_fourth_port_dis_id dikt['port_of_departure'] = itin.port_of_departure_id dikt[ 'number_of_ports_called_prior_to_slave_purchase' @@ -895,6 +901,10 @@ def get_label(obj, field='name'): VoyageCache.ports.get(itin.int_first_port_dis_id)) dikt['second_port_intended_disembarkation_name'] = get_label( VoyageCache.ports.get(itin.int_second_port_dis_id)) + dikt['third_port_intended_disembarkation_name'] = get_label( + VoyageCache.ports.get(itin.int_third_port_dis_id)) + dikt['fourth_port_intended_disembarkation_name'] = get_label( + VoyageCache.ports.get(itin.int_fourth_port_dis_id)) dikt['port_of_departure_name'] = get_label( VoyageCache.ports.get(itin.port_of_departure_id)) dikt['first_place_of_slave_purchase_name'] = get_label( diff --git a/voyages/apps/past/models.py b/voyages/apps/past/models.py index 7fa5354a1..8ec7fe998 100644 --- a/voyages/apps/past/models.py +++ b/voyages/apps/past/models.py @@ -8,6 +8,7 @@ from functools import reduce from datetime import datetime +from django.conf import settings from django.contrib.auth.models import User from django.db import (connection, models, transaction) from django.db.models import (Aggregate, Case, CharField, Count, F, Func, IntegerField, Max, Min, Q, Sum, Value, When) @@ -550,6 +551,72 @@ class EnslaverVoyageConnection(models.Model): order = models.IntegerField(null=True) # NOTE: we will have to substitute VoyageShipOwner and VoyageCaptain # models/tables by this entity. + # leaving this line below for later cleanup when the migration is finished. + # settings.VOYAGE_ENSLAVERS_MIGRATION_STAGE + + +class VoyageCaptainOwnerHelper: + """ + A simple helper class to fetch enslavers associated with a voyage based on + their role. + """ + + _captain_through_table = Voyage.voyage_captain.through._meta.db_table + _owner_through_table = Voyage.voyage_ship_owner.through._meta.db_table + + def __init__(self): + self.owner_role_ids = list( \ + EnslaverRole.objects.filter(name__icontains='owner').values_list('pk', flat=True)) + self.captain_role_ids = list( \ + EnslaverRole.objects.filter(name__icontains='captain').values_list('pk', flat=True)) + + + class UniqueHelper: + """ + A little helper that allows appending to a list only if the items + (names) are unique while maintaining the insertion order. + """ + + def __init__(self): + self.all = [] + self.unique = set() + + def append(self, iterable): + for x in iterable: + if x not in self.unique: + self.unique.add(x) + self.all.append(x) + + + def get_captains(self, voyage): + h = self.UniqueHelper() + if settings.VOYAGE_ENSLAVERS_MIGRATION_STAGE <= 2: + # Fetch from the LEGACY table. + h.append(c.name for c in \ + voyage.voyage_captain.order_by(f"{self.__class__._captain_through_table}.captain_order")) + if settings.VOYAGE_ENSLAVERS_MIGRATION_STAGE >= 2: + # Fetch from EnslaversVoyageConnection. + h.append(self.__class__.get_all_with_roles(voyage, self.captain_role_ids)) + # Dedupe names. + return h.all + + def get_owners(self, voyage): + h = self.UniqueHelper() + if settings.VOYAGE_ENSLAVERS_MIGRATION_STAGE <= 2: + # Fetch from the LEGACY table. + h.append(c.name for c in \ + voyage.voyage_ship_owner.order_by(f"{self.__class__._owner_through_table}.owner_order")) + if settings.VOYAGE_ENSLAVERS_MIGRATION_STAGE >= 2: + # Fetch from EnslaversVoyageConnection. + h.append(self.__class__.get_all_with_roles(voyage, self.owner_role_ids)) + # Dedupe names. + return h.all + + @staticmethod + def get_all_with_roles(voyage, roles): + for ens in voyage.enslavers: + if ens.role_id in roles: + yield ens.enslaver_alias.alias class LanguageGroup(NamedModelAbstractBase): diff --git a/voyages/apps/past/templates/past/_itinerary.html b/voyages/apps/past/templates/past/_itinerary.html index e828d429e..c2f10a365 100644 --- a/voyages/apps/past/templates/past/_itinerary.html +++ b/voyages/apps/past/templates/past/_itinerary.html @@ -27,7 +27,6 @@ diff --git a/voyages/apps/past/views.py b/voyages/apps/past/views.py index 2d56a69e2..547c68432 100644 --- a/voyages/apps/past/views.py +++ b/voyages/apps/past/views.py @@ -572,24 +572,26 @@ def enslaved_contribution(request): name_ids.append(name_entry.pk) result['name_ids'] = name_ids language_ids = [] - for i, lang in enumerate(languages): + print("-->",languages) +# for i, lang in enumerate(languages): + for i in languages: lang_entry = EnslavedContributionLanguageEntry() lang_entry.contribution = contrib lang_entry.order = i + 1 - lang_group_id = lang.get('lang_group_id', None) + lang_group_id = i lang_entry.language_group = LanguageGroup.objects.get( pk=lang_group_id) if lang_group_id else None if lang_entry.language_group is None: transaction.rollback() return HttpResponseBadRequest( 'Invalid language entry in contribution') - modern_country_id = lang.get('modern_country_id', None) - lang_entry.modern_country = ModernCountry.objects.get( - pk=modern_country_id) if modern_country_id else None - if modern_country_id is None: - transaction.rollback() - return HttpResponseBadRequest( - 'Invalid modern country entry in contribution') +# modern_country_id = lang.get('modern_country_id', None) +# lang_entry.modern_country = ModernCountry.objects.get( +# pk=modern_country_id) if modern_country_id else None +# if modern_country_id is None: +# transaction.rollback() +# return HttpResponseBadRequest( +# 'Invalid modern country entry in contribution') lang_entry.save() language_ids.append(lang_entry.pk) result['language_ids'] = language_ids diff --git a/voyages/apps/voyage/management/commands/importcsv.py b/voyages/apps/voyage/management/commands/importcsv.py index a5bb3e79d..9a0ada81a 100644 --- a/voyages/apps/voyage/management/commands/importcsv.py +++ b/voyages/apps/voyage/management/commands/importcsv.py @@ -8,7 +8,6 @@ from django.utils.encoding import smart_str from voyages.apps.contribute.publication import CARGO_COLUMN_COUNT -from voyages.apps.resources.models import AfricanName, Image from voyages.apps.voyage.models import (AfricanInfo, BroadRegion, CargoType, CargoUnit, LinkedVoyages, Nationality, OwnerOutcome, ParticularOutcome, Place, Region, @@ -24,6 +23,104 @@ VoyageSourcesConnection) from voyages.apps.common.utils import * +import re + +def _migrate_enslavers_from_legacy(): + from django.conf import settings + if settings.VOYAGE_ENSLAVERS_MIGRATION_STAGE <= 1: + # Not really necessary but it is convenient to reference the stage + # settings variable so that we can later remove dead code paths once the + # migration is finalized. + return [] + # Try to match each Captain/Owner with an existing EnslaverAlias. If none + # are found, we create the alias and the identity for it. In case of + # duplicate identities with same alias already exist, check if one of them + # is already associated with the existing voyage. + from voyages.apps.past.models import EnslaverAlias, EnslaverIdentity, EnslaverVoyageConnection, VoyageCaptainOwnerHelper + from unidecode import unidecode + aliases = {} + for alias in EnslaverAlias.objects.all(): + aliases.setdefault(unidecode(alias.alias), []).append((alias.id, alias.identity_id)) + connections = {} + for conn in EnslaverVoyageConnection.objects.all(): + connections.setdefault(conn.enslaver_alias_id, []).append((conn.voyage_id, conn.role_id)) + + def get_actions(name_and_voyages, role_ids): + migration_actions = [] + for name, voyage_id in name_and_voyages: + matches = aliases.get(unidecode(name)) + item = { + 'voyage_id': voyage_id, + 'name': name, + 'reason': '', + 'steps': [] + } + found = False + if matches: + for alias_id, _ in matches: + if voyage_id in [x[0] for x in connections.get(alias_id, []) if x[1] in role_ids]: + # Happy path! + # The appropriate voyage connection already exists. + found = True + item['reason'] = 'Exact match for voyage, name, and role in ' + EnslaverVoyageConnection.__name__ + break + if not found: + if len({x[1] for x in matches}) > 1: + # Not so good path. + # There are multiple identities maching this name so we + # cannot determine for sure which is the right one. + item['reason'] = 'Multiple identities containing the same alias' + else: + # OK path + # We only need to create the connection for the existing alias. + item['reason'] = 'Found a single matching identity with the alias' + item['steps'].append({ + 'type': EnslaverVoyageConnection.__name__, + 'values': { + 'enslaver_alias_id': matches[0][0], + 'voyage_id': voyage_id, + } + }) + # Treat this as a found alias. + found = True + else: + item['reason'] = 'No match found for enslaver alias' + if not found: + item['steps'].append({ + 'type': EnslaverIdentity.__name__, + 'values': { + 'principal_alias': name + } + }) + item['steps'].append({ + 'type': EnslaverAlias.__name__, + 'values': { + 'alias': name + } + }) + migration_actions.append(item) + return migration_actions + + helper = VoyageCaptainOwnerHelper() + actions = get_actions( + [[cc.captain.name, cc.voyage_id] for cc in \ + VoyageCaptainConnection.objects.select_related('captain').all()], + helper.captain_role_ids + ) + actions += get_actions( + [[oc.owner.name, oc.voyage_id] for oc in \ + VoyageShipOwnerConnection.objects.select_related('owner').all()], + helper.owner_role_ids + ) + # Usage in django shell + # from voyages.apps.voyage.management.commands import importcsv + # actions = importcsv._migrate_enslavers_from_legacy() + # for a in actual: + # by_reason.setdefault(a['reason'], []).append(a) + # [(k, len(v)) for k, v in by_reason.items()] + return actions + + class Command(BaseCommand): help = ('Imports a CSV file with the full Voyages dataset and converts the data ' 'to the Django models.') @@ -68,10 +165,6 @@ def handle(self, csv_files, *args, **options): voyage_dates = [] voyage_numbers = [] - # Prefetch data: Miscellaneous - africans = list(AfricanName.objects.all()) - images = list(Image.objects.all()) - def check_hierarchy(voyage_id, field, place, region, broad_region): """ Checks that the imported values match the geo database. @@ -118,9 +211,10 @@ def get_component(suffix, expected_len): return component if suffixes is None: suffixes = ['a', 'b', 'c'] - return get_component(suffixes[1], 2) + ',' + \ - get_component(suffixes[0], 2) + ',' + \ - get_component(suffixes[2], 4) + month = get_component(suffixes[1], 2) + day = get_component(suffixes[0], 2) + year = get_component(suffixes[2], 4) + return f"{month},{day},{year}" if month or day or year else '' def date_from_sep_values(value): """ @@ -162,7 +256,7 @@ def get_component_val(cval, min_value, max_value, mandatory): day = get_component_val(components[coords[2]], 1, 31, False) month = get_component_val(components[coords[1]], 1, 12, bool(day)) year = get_component_val(components[coords[0]], 999, 1900, bool(month)) - return f'{month},{day},{year}' + return f'{month},{day},{year}' if month or day or year else '' except Exception as e: self.errors += 1 error_reporting.report('Invalid date "' + value + '" ' + str(e)) @@ -210,6 +304,10 @@ def row_to_itinerary(row, voyage): Place, 'arrport') itinerary.int_second_port_dis = row.get_by_value( Place, 'arrport2') + itinerary.int_third_port_dis = row.get_by_value( + Place, 'arrport3') + itinerary.int_fourth_port_dis = row.get_by_value( + Place, 'arrport4') itinerary.int_first_region_slave_landing = row.get_by_value( Region, 'regarr') itinerary.int_second_place_region_slave_landing = ( @@ -495,7 +593,7 @@ def row_to_numbers(row, voyage): voyage.voyage_in_cd_rom = in_cd_room == '1' voyage.voyage_groupings = rh.get_by_value(VoyageGroupings, 'xmimpflag') voyage.dataset = rh.cint('dataset', allow_null=False) - voyage.comments = rh.get('COMMENTS') + voyage.comments = rh.get('comments') intra_american = voyage.dataset == 1 counts[voyage.dataset] = counts.get(voyage.dataset, 0) + 1 ships.append(row_to_ship(rh, voyage)) @@ -504,8 +602,7 @@ def row_to_numbers(row, voyage): crews.append(row_to_crew(rh, voyage)) voyage_numbers.append(row_to_numbers(rh, voyage)) # Captains - order = 1 - for key in get_multi_valued_column_suffix(3): + for order, key in enumerate(get_multi_valued_column_suffix(3)): captain_name = rh.get('captain' + key) if captain_name is None or empty.match(captain_name): continue @@ -516,13 +613,11 @@ def row_to_numbers(row, voyage): captains[captain_name] = captain_model captain_connection = VoyageCaptainConnection() captain_connection.captain = captain_model - captain_connection.captain_order = order + captain_connection.captain_order = order + 1 captain_connection.voyage = voyage captain_connections.append(captain_connection) - order += 1 # Ship owners - order = 1 - for key in get_multi_valued_column_suffix(45): + for order, key in enumerate(get_multi_valued_column_suffix(45)): owner_name = rh.get('owner' + key) if owner_name is None or empty.match(owner_name): continue @@ -533,13 +628,11 @@ def row_to_numbers(row, voyage): ship_owners[owner_name] = owner_model owner_connection = VoyageShipOwnerConnection() owner_connection.owner = owner_model - owner_connection.owner_order = order + owner_connection.owner_order = order + 1 owner_connection.voyage = voyage ship_owner_connections.append(owner_connection) - order += 1 # Sources - order = 1 - for key in get_multi_valued_column_suffix(18): + for order, key in enumerate(get_multi_valued_column_suffix(18)): source_ref = rh.get('source' + key) if source_ref is None or empty.match(source_ref): break @@ -557,10 +650,9 @@ def row_to_numbers(row, voyage): source_connection = VoyageSourcesConnection() source_connection.group = voyage source_connection.source = source - source_connection.source_order = order + source_connection.source_order = order + 1 source_connection.text_ref = source_ref source_connections.append(source_connection) - order += 1 # African info for key in get_multi_valued_column_suffix(3): afrinfoval = rh.get_by_value(AfricanInfo, 'afrinfo' + key, key_name='id') @@ -593,14 +685,29 @@ def row_to_numbers(row, voyage): outcome.voyage = voyage outcomes.append(outcome) # Links - if intra_american and 'voyageid2' in rh.row: - voyage_links.append( - (voyage_id, rh.cint('voyageid2'), - LinkedVoyages.INTRA_AMERICAN_LINK_MODE)) + if 'voyageid2' in rh.row: + link_mode = LinkedVoyages.INTRA_AMERICAN_LINK_MODE if intra_american else LinkedVoyages.UNSPECIFIED + for entry in re.split(',|;|/', rh.get('voyageid2')): + if not entry: + continue + try: + voyage_links.append( + (voyage_id, int(entry), link_mode)) + except: + print(f"Bad link value {entry}") print('Constructed ' + str(len(voyages)) + ' voyages from CSV') for k, v in counts.items(): print('Dataset ' + str(k) + ': ' + str(v)) + + previous_ids = set(Voyage.all_dataset_objects.values_list('voyage_id', flat=True)) + next_ids = set(voyages.keys()) + missing = previous_ids - next_ids + if len(missing) > 0: + self.errors += 1 + error_reporting.report(f"There are {len(missing)} existing voyages without replacement!!") + print("Missing voyage ids:") + print(missing) if self.errors > 0: print( str(self.errors) + ' errors occurred, ' @@ -648,8 +755,6 @@ def clear_fk(fk_field): helper.delete_all(cursor, VoyageSlavesNumbers) helper.delete_all(cursor, Voyage.african_info.through) helper.delete_all(cursor, VoyageCargoConnection) - helper.delete_all(cursor, AfricanName) - helper.delete_all(cursor, Image) helper.delete_all(cursor, Voyage) print('Inserting new records...') @@ -703,8 +808,6 @@ def set_voyages_fk(items): bulk_insert(VoyageCrew, crews) bulk_insert(VoyageSlavesNumbers, voyage_numbers) bulk_insert(VoyageOutcome, outcomes) - bulk_insert(AfricanName, africans) - bulk_insert(Image, images) # Now insert the many-to-many connections set_voyages_fk(captain_connections) diff --git a/voyages/apps/voyage/migrations/0020_auto_20230131_1336.py b/voyages/apps/voyage/migrations/0020_auto_20230131_1336.py new file mode 100644 index 000000000..71f25cff1 --- /dev/null +++ b/voyages/apps/voyage/migrations/0020_auto_20230131_1336.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.17 on 2023-01-31 13:36 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('voyage', '0019_africaninfo_possibly_offensive'), + ] + + operations = [ + migrations.AddField( + model_name='voyageitinerary', + name='int_fourth_place_region_slave_landing', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='int_fourth_region_slave_landing', to='voyage.Region', verbose_name='Fourth intended region of slave landing (REGARR4)'), + ), + migrations.AddField( + model_name='voyageitinerary', + name='int_fourth_port_dis', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='int_fourth_port_dis', to='voyage.Place', verbose_name='Fourth intended port of disembarkation (ARRPORT4)'), + ), + migrations.AddField( + model_name='voyageitinerary', + name='int_third_place_region_slave_landing', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='int_third_region_slave_landing', to='voyage.Region', verbose_name='Third intended region of slave landing (REGARR3)'), + ), + migrations.AddField( + model_name='voyageitinerary', + name='int_third_port_dis', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='int_third_port_dis', to='voyage.Place', verbose_name='Third intended port of disembarkation (ARRPORT3)'), + ), + ] diff --git a/voyages/apps/voyage/models.py b/voyages/apps/voyage/models.py index 098321e7a..2ae357e50 100644 --- a/voyages/apps/voyage/models.py +++ b/voyages/apps/voyage/models.py @@ -9,6 +9,8 @@ from voyages.apps.common.validators import date_csv_field_validator +from django.conf import settings + class AfricanInfo(NamedModelAbstractBase): """ Used to capture information about the ethnicity or background of the @@ -553,6 +555,20 @@ class VoyageItinerary(models.Model): null=True, blank=True, on_delete=models.CASCADE) + int_third_port_dis = models.ForeignKey( + 'Place', + related_name="int_third_port_dis", + verbose_name="Third intended port of disembarkation (ARRPORT3)", + null=True, + blank=True, + on_delete=models.CASCADE) + int_fourth_port_dis = models.ForeignKey( + 'Place', + related_name="int_fourth_port_dis", + verbose_name="Fourth intended port of disembarkation (ARRPORT4)", + null=True, + blank=True, + on_delete=models.CASCADE) int_first_region_slave_landing = models.ForeignKey( 'Region', related_name="int_first_region_slave_landing", @@ -567,6 +583,20 @@ class VoyageItinerary(models.Model): null=True, blank=True, on_delete=models.CASCADE) + int_third_place_region_slave_landing = models.ForeignKey( + 'Region', + related_name="int_third_region_slave_landing", + verbose_name="Third intended region of slave landing (REGARR3)", + null=True, + blank=True, + on_delete=models.CASCADE) + int_fourth_place_region_slave_landing = models.ForeignKey( + 'Region', + related_name="int_fourth_region_slave_landing", + verbose_name="Fourth intended region of slave landing (REGARR4)", + null=True, + blank=True, + on_delete=models.CASCADE) # End of intended variables ports_called_buying_slaves = models.IntegerField( @@ -1812,6 +1842,8 @@ def __str__(self): def __unicode__(self): return str(self.first) + " => " + str(self.second) + UNSPECIFIED = 0 + # In this mode the first voyage is the IntraAmerican voyage # and the second is a transatlantic voyage. INTRA_AMERICAN_LINK_MODE = 1 @@ -1872,15 +1904,16 @@ class Voyage(models.Model): related_name='voyage_slaves_numbers', on_delete=models.CASCADE) - voyage_captain = models.ManyToManyField("VoyageCaptain", - through='VoyageCaptainConnection', - help_text="Voyage Captain", - blank=True) - voyage_ship_owner = models.ManyToManyField( - "VoyageShipOwner", - through='VoyageShipOwnerConnection', - help_text="Voyage Ship Owner", - blank=True) + if settings.VOYAGE_ENSLAVERS_MIGRATION_STAGE <= 2: + voyage_captain = models.ManyToManyField("VoyageCaptain", + through='VoyageCaptainConnection', + help_text="Voyage Captain", + blank=True) + voyage_ship_owner = models.ManyToManyField( + "VoyageShipOwner", + through='VoyageShipOwnerConnection', + help_text="Voyage Ship Owner", + blank=True) # One Voyage can contain multiple sources and one source can refer # to multiple voyages @@ -1954,10 +1987,14 @@ def __init__(self): 'imp_principal_port_slave_dis__region__broad_region', 'imp_principal_region_of_slave_purchase', 'imp_principal_region_slave_dis', 'imp_region_voyage_begin', - 'int_first_port_dis', 'int_first_port_emb', + 'int_first_port_dis', 'int_second_port_dis', + 'int_third_port_dis', 'int_fourth_port_dis', + 'int_first_port_emb', 'int_first_region_purchase_slaves', 'int_first_region_slave_landing', - 'int_second_place_region_slave_landing', 'int_second_port_dis', + 'int_second_place_region_slave_landing', + 'int_third_place_region_slave_landing', + 'int_fourth_place_region_slave_landing', 'int_second_port_emb', 'int_second_region_purchase_slaves', 'place_voyage_ended', 'port_of_call_before_atl_crossing', 'port_of_departure', 'principal_place_of_slave_purchase', @@ -1969,9 +2006,6 @@ def __init__(self): ] self.prefetch_fields = [ - Prefetch('voyage_captain', - queryset=VoyageCaptain.objects.order_by( - 'captain_name__captain_order')), Prefetch('voyage_ship__nationality_ship'), Prefetch('voyage_ship__imputed_nationality'), Prefetch('voyage_ship__ton_type'), @@ -1980,9 +2014,6 @@ def __init__(self): Prefetch('voyage_ship__vessel_construction_region'), Prefetch('voyage_ship__registered_place'), Prefetch('voyage_ship__registered_region'), - Prefetch('voyage_ship_owner', - queryset=VoyageShipOwner.objects.order_by( - 'owner_name__owner_order')), Prefetch('african_info'), Prefetch('cargo', queryset=VoyageCargoConnection.objects.prefetch_related('cargo').prefetch_related('unit')), @@ -2005,7 +2036,20 @@ def __init__(self): queryset=LinkedVoyages.objects.select_related('first').only( 'first_id', 'second_id', 'first__voyage_id')) ] - + if settings.VOYAGE_ENSLAVERS_MIGRATION_STAGE <= 2: + self.prefetch_fields += [ + Prefetch('voyage_captain', + queryset=VoyageCaptain.objects.order_by( + 'captain_name__captain_order')), + Prefetch('voyage_ship_owner', + queryset=VoyageShipOwner.objects.order_by( + 'owner_name__owner_order'))] + if settings.VOYAGE_ENSLAVERS_MIGRATION_STAGE >= 2: + from voyages.apps.past.models import EnslaverVoyageConnection + self.prefetch_fields.append( + Prefetch('enslavervoyageconnection_set', + queryset=EnslaverVoyageConnection.objects.order_by('order').prefetch_related('enslaver_alias', 'role'), + to_attr='enslavers')) for k, v in list(self.related_models.items()): self.prefetch_fields += [ Prefetch(k + '__' + f.name, diff --git a/voyages/apps/voyage/search_indexes.py b/voyages/apps/voyage/search_indexes.py index afca43bdb..e25e5297d 100644 --- a/voyages/apps/voyage/search_indexes.py +++ b/voyages/apps/voyage/search_indexes.py @@ -16,6 +16,8 @@ from .globals import no_mangle, search_mangle_methods from .models import Voyage, VoyagesFullQueryHelper, VoyageSources +from voyages.apps.past.models import VoyageCaptainOwnerHelper + def split_date(value): if value is None: @@ -72,6 +74,8 @@ def mkdate(year, month, day): return date(year, month, day) +_captain_owner_helper = VoyageCaptainOwnerHelper() + # Index for Sources @@ -739,6 +743,11 @@ def get_updated_field(self): def index_queryset(self, using=None): """Used when the entire index for model is updated.""" + # Note: it is very important to use this helper for several reasons. The + # main one is performance since the helper ensures that lots of data are + # prefetched and joined, saving a huge number of calls to the database. + # The other reason is that it maps some relations with aliases and + # indexing methods may rely on these aliases to get the relational data. helper = VoyagesFullQueryHelper() return helper.get_query() @@ -750,7 +759,7 @@ def prepare_var_imp_voyage_began(self, obj): def prepare_var_owner(self, obj): try: - return '
'.join([o.name for o in obj.voyage_ship_owner.all()]) + return '
'.join(_captain_owner_helper.get_owners(obj)) except AttributeError: return None @@ -887,8 +896,7 @@ def prepare_var_length_middle_passage_days(self, obj): # Voyage crew def prepare_var_captain(self, obj): - return '
'.join( - [captain.name for captain in obj.voyage_captain.all()]) + return '
'.join(_captain_owner_helper.get_captains(obj)) def prepare_var_captain_plaintext(self, obj): return self.prepare_var_captain(obj) diff --git a/voyages/apps/voyage/search_views.py b/voyages/apps/voyage/search_views.py index 3c2df514e..f194d7e87 100644 --- a/voyages/apps/voyage/search_views.py +++ b/voyages/apps/voyage/search_views.py @@ -646,6 +646,7 @@ def ajax_search(request): "Voyage ID", "var_year_of_construction": "Year constructed", + "var_voyage_links": pgettext_lazy("datatable column header", "VOYAGEID2"), } diff --git a/voyages/settings.py b/voyages/settings.py index b9b8e91d2..9a06d7567 100644 --- a/voyages/settings.py +++ b/voyages/settings.py @@ -243,7 +243,19 @@ def is_feature_enabled(feature_name): return FEATURE_FLAGS.get(feature_name, False) +# The migration path for Voyage Captains and Owners will go through 4 stages: +# 1 - Legacy and Enslaver tables coexist but only Legacy shows up in the Voyages +# end and are imported/exported in CSV format. +# 2 - Both tables coexist and the records are combined in the front-end and +# import/exports. Duplication in the db is expected but we will dedupe the Solr +# index and CSV export. +# 3 - The legacy tables are kept but their results are no longer visible in the +# UI and are no longer imported/exported in CSV format. +# 4 - After enough testing on stage 3, the legacy tables will be removed from +# the db. + +VOYAGE_ENSLAVERS_MIGRATION_STAGE = 1 # Modify HAYSTACK config for fixture loading durring tests @@ -268,4 +280,3 @@ def is_feature_enabled(feature_name): file=sys.stderr) del sys - diff --git a/voyages/sitemedia/scripts/vue/past-contribute/app.js b/voyages/sitemedia/scripts/vue/past-contribute/app.js index 8a7e22a0e..25b9ba422 100644 --- a/voyages/sitemedia/scripts/vue/past-contribute/app.js +++ b/voyages/sitemedia/scripts/vue/past-contribute/app.js @@ -116,12 +116,12 @@ var pastContribute = new Vue({ var contrib_languages = []; if (Array.isArray(this.language_groups.value.searchTerm)) { $.each(this.language_groups.value.searchTerm, function(key, value){ - contrib_languages.push(get_lang_contrib_item(value)); + contrib_languages.push(value); }); } else { var value = this.language_groups.value.searchTerm; value = value.slice(value.indexOf('-') + 1); - contrib_languages.push(get_lang_contrib_item(value)); + contrib_languages.push(value); } var params = {