Feature/importation (#183)

* Initial planning and work on Voyages' enslaver migration.
We are splitting the process in four steps to allow for ample testing and verification.
In the intermediary steps, both legacy VoyageCaptain and VoyageOwner tables will be used alongside the Enslaver (PAST) data.
A settings.py variable was created to ensure that we can advance or rollback on the steps.
Once finalized, the plan is to drop the legacy tables.

* Additional code to help with data migration Voyages to PAST.

* Adding two extra intended disembarkation ports to VoyageItinerary.
Guarding None values in _map_voyage_to_spss.
Fixing bugs in export of AFRINFO / CARGO vars.

* Disembarkation ports now selectable in OoK.

* Fixing issues in export and import CSV for voyages.
A roundtrip test: export to file 1, import from file 1, export to file 2
yielded identical files 1 and 2.

* Fixing DataTable CSV/Excel export.
Issue was new Linked Voyage column which contains array-valued cells.

* pip and mysql library version bumps

* removing moderncountry from enslaved language group contribution (as it is implicit in the lg)

---------

Co-authored-by: Domingos <[email protected]>
Co-authored-by: John Mulligan <[email protected]>

.gitignore

@@ -21,3 +21,4 @@ google_auth.json
 documents/audio/**
 documents/blog/**
 documents/_versions/**
+documents/csv_downloads/**

docker/django/Dockerfile

@@ -9,10 +9,10 @@ FROM ubuntu:18.04 AS base
 RUN apt-get update -y \
   && apt-get install --yes --no-upgrade --no-install-recommends \
     gettext=0.19.8.1-6ubuntu0.3 \
-    mysql-client=5.7.40-0ubuntu0.18.04.1 \
-    libmysqlclient-dev=5.7.40-0ubuntu0.18.04.1 \
+    mysql-client=5.7.41-0ubuntu0.18.04.1 \
+    libmysqlclient-dev=5.7.41-0ubuntu0.18.04.1 \
     python3=3.6.7-1~18.04 \
-    python3-pip=9.0.1-2.3~ubuntu1.18.04.5 \
+    python3-pip=9.0.1-2.3~ubuntu1.18.04.6 \
     python3-future=0.15.2-4ubuntu2 \
     python3-pygit2=0.26.2-2 \
   && apt-get clean \

voyages/apps/common/export.py

@@ -2,10 +2,13 @@
 
 from builtins import str
 
+import json
 import xlwt
+from collections.abc import Iterable
 from django.http import HttpResponse
 
 
+
 def download_xls(header_rows, data_set, row_header_columns=None):
     """
     Generates an XLS file with the given data.
@@ -52,7 +55,8 @@ def download_xls(header_rows, data_set, row_header_columns=None):
     # Write tabular data.
     for row in data_set:
         # TODO: use XLSX format that allows more rows!
-        if row_index == 65536:
+        if row_index == 65535:
+            ws.write(row_index, 0, 'Output is truncated!')
             break
         col_index = 0
         for rhd in row_header_data:
@@ -64,6 +68,8 @@ def download_xls(header_rows, data_set, row_header_columns=None):
                              col_index)
             col_index += 1
         for cell in row:
+            if not isinstance(cell, str) and isinstance(cell, Iterable):
+                cell = json.dumps(cell)
             ws.write(row_index, col_index, cell, number_style)
             col_index += 1
         row_index += 1

voyages/apps/common/utils.py

@@ -8,17 +8,18 @@
 
 empty = re.compile(r"^\s*\.?$")
 
-def get_multi_valued_column_suffix(max_columns):
-    ALPHABET = 26
-    if max_columns > 2 * ALPHABET: raise Exception("Too many columns!")
-    first_char = ord('a')
-    single_char_limit = min(max_columns, ALPHABET)
+def get_multi_valued_column_suffix(max_columns, upper_case=False):
+    ALPHABET_LEN = 26
+    if max_columns > 2 * ALPHABET_LEN: raise Exception("Too many columns!")
+    start_char = 'A' if upper_case else 'a'
+    first_char = ord(start_char)
+    single_char_limit = min(max_columns, ALPHABET_LEN)
     for i in range(0, single_char_limit):
         yield chr(first_char + i)
-    max_columns -= ALPHABET
+    max_columns -= ALPHABET_LEN
     if max_columns > 0:
         for i in range(0, max_columns):
-            yield 'a' + chr(first_char + i)
+            yield start_char + chr(first_char + i)
 
 class RowHelper:
     """

voyages/apps/contribute/forms.py

@@ -210,6 +210,10 @@ class Meta:
             _('First port of intended disembarkation'),
             'second_port_intended_disembarkation':
             _('Second port of intended disembarkation'),
+            'third_port_intended_disembarkation':
+            _('Third port of intended disembarkation'),
+            'fourth_port_intended_disembarkation':
+            _('Fourth port of intended disembarkation'),
             'port_of_departure': _('Port of departure'),
             'number_of_ports_called_prior_to_slave_purchase':
             _('Number of ports called prior to slave purchase'),

voyages/apps/contribute/migrations/0024_auto_20230131_1336.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.17 on 2023-01-31 13:36
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('voyage', '0020_auto_20230131_1336'),
+        ('contribute', '0023_auto_20220621_1812'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='interimvoyage',
+            name='fourth_port_intended_disembarkation',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='voyage.Place'),
+        ),
+        migrations.AddField(
+            model_name='interimvoyage',
+            name='third_port_intended_disembarkation',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='voyage.Place'),
+        ),
+    ]

voyages/apps/contribute/models.py

@@ -127,6 +127,18 @@ class InterimVoyage(models.Model):
         null=True,
         blank=True,
         on_delete=models.CASCADE)
+    third_port_intended_disembarkation = models.ForeignKey(
+        voyage.models.Place,
+        related_name='+',
+        null=True,
+        blank=True,
+        on_delete=models.CASCADE)
+    fourth_port_intended_disembarkation = models.ForeignKey(
+        voyage.models.Place,
+        related_name='+',
+        null=True,
+        blank=True,
+        on_delete=models.CASCADE)
     port_of_departure = models.ForeignKey(
         voyage.models.Place,
         related_name='+',