combo/apps/lingo/migrations/0048_remote_items_text_to_array.py

@@ -0,0 +1,25 @@
+from django.contrib.postgres.fields import ArrayField
+from django.contrib.postgres.indexes import GinIndex
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('lingo', '0047_transaction_index_startdate'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            """ALTER TABLE lingo_transaction
+            ALTER COLUMN remote_items TYPE character varying(512)[] USING string_to_array(remote_items, ','),
+            ALTER COLUMN to_be_paid_remote_items TYPE character varying(512)[] USING string_to_array(to_be_paid_remote_items, ',');
+        """
+        ),
+        migrations.AlterField(
+            'transaction', 'remote_items', ArrayField(models.CharField(max_length=512), null=True)
+        ),
+        migrations.AlterField(
+            'transaction', 'to_be_paid_remote_items', ArrayField(models.CharField(max_length=512), null=True)
+        ),
+        migrations.AddIndex('transaction', GinIndex(name='remote_items__gin_idx', fields=['remote_items'])),
+    ]

combo/apps/lingo/models.py

@@ -27,6 +27,8 @@ import eopayment
 from dateutil import parser
 from django import template
 from django.conf import settings
+from django.contrib.postgres.fields import ArrayField
+from django.contrib.postgres.indexes import GinIndex
 from django.core import serializers
 from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
 from django.core.mail import EmailMultiAlternatives
@@ -756,12 +758,12 @@ class RemoteItem:
         # filter transactions by regie, status and contained remote_item id
         query = reduce(
             models.Q.__or__,
-            (models.Q(remote_items__contains=remote_item_id) for remote_item_id in remote_item_ids),
+            (models.Q(remote_items__contains=[remote_item_id]) for remote_item_id in remote_item_ids),
         )
 
         # accumulate in paid_items each remote_item earliest payment_date
         for transaction in queryset.filter(query):
-            for remote_item_id in transaction.remote_items.split(','):
+            for remote_item_id in transaction.remote_items:
                 if remote_item_id in remote_item_ids:
                     yield transaction
                     break
@@ -778,7 +780,7 @@ class RemoteItem:
         for transaction in cls.transactions_for_remote_items(transaction_qs, remote_items):
             if transaction.is_running() and can_poll_backend:
                 transaction.poll_backend()
-            for remote_item in transaction.remote_items.split(','):
+            for remote_item in transaction.remote_items:
                 if transaction.end_date and transaction.is_paid():
                     if remote_item not in paid_items:
                         paid_items[remote_item] = transaction.end_date
@@ -812,8 +814,8 @@ def status_label(status):
 class Transaction(models.Model):
     regie = models.ForeignKey(Regie, on_delete=models.CASCADE, null=True)
     items = models.ManyToManyField(BasketItem, blank=True)
-    remote_items = models.CharField(max_length=512)
-    to_be_paid_remote_items = models.CharField(max_length=512, null=True)
+    remote_items = ArrayField(models.CharField(max_length=512), null=True)
+    to_be_paid_remote_items = ArrayField(models.CharField(max_length=512), null=True)
     start_date = models.DateTimeField(auto_now_add=True, db_index=True)
     end_date = models.DateTimeField(null=True)
     bank_data = JSONField(blank=True, default=dict)
@@ -827,6 +829,11 @@ class Transaction(models.Model):
     RUNNING_STATUSES = [0, eopayment.WAITING, eopayment.RECEIVED]
     PAID_STATUSES = [eopayment.PAID, eopayment.ACCEPTED]
 
+    class Meta:
+        indexes = [
+            GinIndex(name='remote_items__gin_idx', fields=['remote_items']),
+        ]
+
     def is_remote(self):
         return self.remote_items != ''
 
@@ -858,7 +865,7 @@ class Transaction(models.Model):
 
         regie = self.regie
         to_be_paid_remote_items = []
-        for item_id in items.split(','):
+        for item_id in items:
             try:
                 remote_item = regie.get_invoice(user=self.user, invoice_id=item_id, raise_4xx=True)
                 with atomic(savepoint=False):
@@ -886,7 +893,7 @@ class Transaction(models.Model):
             else:
                 logger.info('notified payment for remote item %s from transaction %s', item_id, self)
 
-        self.to_be_paid_remote_items = ','.join(to_be_paid_remote_items) or None
+        self.to_be_paid_remote_items = to_be_paid_remote_items or None
         self.save(update_fields=['to_be_paid_remote_items'])
 
     def create_paid_invoice_basket_item(self, item_id, remote_item):

combo/apps/lingo/views.py

@@ -414,7 +414,7 @@ class PayMixin:
         transaction.save()
         transaction.regie = regie
         transaction.items.set(items)
-        transaction.remote_items = ','.join([x.id for x in remote_items])
+        transaction.remote_items = [x.id for x in remote_items]
         transaction.status = 0
         transaction.amount = total_amount
 

tests/test_lingo_manager.py

@@ -180,7 +180,7 @@ def test_download_transaction(app, admin_user, regie):
     )
     trans1 = Transaction.objects.create(
         regie=regie,
-        remote_items='remote items lol',
+        remote_items=['remote items lol'],
         order_id='1',
         user=user,
         bank_transaction_id='567',
@@ -188,7 +188,7 @@ def test_download_transaction(app, admin_user, regie):
     )
     trans2 = Transaction.objects.create(
         regie=regie,
-        remote_items='remote items omg',
+        remote_items=['remote items omg'],
         order_id='2',
         user=user,
         bank_transaction_id='136',

tests/test_lingo_payment.py

@@ -1140,7 +1140,10 @@ def test_transaction_expiration():
 @mock.patch('combo.utils.requests_wrapper.RequestsSession.request')
 def test_transaction_retry(mock_request, remote_regie):
     transaction = Transaction.objects.create(
-        status=eopayment.PAID, regie=remote_regie, end_date=timezone.now(), to_be_paid_remote_items='42,35'
+        status=eopayment.PAID,
+        regie=remote_regie,
+        end_date=timezone.now(),
+        to_be_paid_remote_items=['42', '35'],
     )
     transaction.start_date = timezone.now() - timedelta(days=3)
     transaction.save()
@@ -1165,19 +1168,22 @@ def test_transaction_retry(mock_request, remote_regie):
     assert transaction.to_be_paid_remote_items is None
 
     # too old
-    transaction.to_be_paid_remote_items = '42,35'
+    transaction.to_be_paid_remote_items = ['42', '35']
     transaction.start_date = timezone.now() - timedelta(days=4)
     transaction.save()
 
     appconfig.update_transactions()
     transaction.refresh_from_db()
-    assert transaction.to_be_paid_remote_items == '42,35'
+    assert transaction.to_be_paid_remote_items == ['42', '35']
 
 
 @mock.patch('combo.utils.requests_wrapper.RequestsSession.request')
 def test_transaction_retry_failure(mock_request, remote_regie):
     transaction = Transaction.objects.create(
-        status=eopayment.PAID, regie=remote_regie, end_date=timezone.now(), to_be_paid_remote_items='42,35'
+        status=eopayment.PAID,
+        regie=remote_regie,
+        end_date=timezone.now(),
+        to_be_paid_remote_items=['42', '35'],
     )
 
     appconfig = apps.get_app_config('lingo')
@@ -1215,10 +1221,10 @@ def test_transaction_retry_failure(mock_request, remote_regie):
     transaction.refresh_from_db()
     assert transaction.items.count() == 1  # only 35 was found
     assert set(transaction.items.values_list('remote_item_id', flat=True)) == {'35'}
-    assert transaction.to_be_paid_remote_items == '42'  # retry for first one
+    assert transaction.to_be_paid_remote_items == ['42']  # retry for first one
 
     # error on pay invoice
-    Transaction.objects.update(to_be_paid_remote_items='42,35')
+    Transaction.objects.update(to_be_paid_remote_items=['42', '35'])
     mock_request.side_effect = [
         mock_json_item,  # get invoice 42
         ConnectionError('where is my hostname?'),  # pay invoice 42
@@ -1229,10 +1235,10 @@ def test_transaction_retry_failure(mock_request, remote_regie):
     transaction.refresh_from_db()
     assert transaction.items.count() == 2  # both were updated now that get_invoice worked for 42
     assert set(transaction.items.values_list('remote_item_id', flat=True)) == {'35', '42'}
-    assert transaction.to_be_paid_remote_items == '42'  # retry for first one
+    assert transaction.to_be_paid_remote_items == ['42']  # retry for first one
 
     # unknown error on get invoice
-    Transaction.objects.update(to_be_paid_remote_items='42,35')
+    Transaction.objects.update(to_be_paid_remote_items=['42', '35'])
     mock_request.side_effect = [
         Exception,  # get invoice 42
         mock_json_item,  # get invoice 35
@@ -1240,10 +1246,10 @@ def test_transaction_retry_failure(mock_request, remote_regie):
     ]
     appconfig.update_transactions()
     transaction.refresh_from_db()
-    assert transaction.to_be_paid_remote_items == '42'  # retry for first one
+    assert transaction.to_be_paid_remote_items == ['42']  # retry for first one
 
     # unknown error on pay invoice
-    Transaction.objects.update(to_be_paid_remote_items='42,35')
+    Transaction.objects.update(to_be_paid_remote_items=['42', '35'])
     mock_request.side_effect = [
         mock_json_item,  # get invoice 42
         Exception,  # pay invoice 42
@@ -1252,10 +1258,10 @@ def test_transaction_retry_failure(mock_request, remote_regie):
     ]
     appconfig.update_transactions()
     transaction.refresh_from_db()
-    assert transaction.to_be_paid_remote_items == '42'  # retry for first one
+    assert transaction.to_be_paid_remote_items == ['42']  # retry for first one
 
     # 5xx on get invoice
-    Transaction.objects.update(to_be_paid_remote_items='42,35')
+    Transaction.objects.update(to_be_paid_remote_items=['42', '35'])
     mock_request.side_effect = [
         mock_5xx,  # get invoice 42
         mock_json_item,  # get invoice 35
@@ -1263,10 +1269,10 @@ def test_transaction_retry_failure(mock_request, remote_regie):
     ]
     appconfig.update_transactions()
     transaction.refresh_from_db()
-    assert transaction.to_be_paid_remote_items == '42'  # retry for first one
+    assert transaction.to_be_paid_remote_items == ['42']  # retry for first one
 
     # 5xx on pay invoice
-    Transaction.objects.update(to_be_paid_remote_items='42,35')
+    Transaction.objects.update(to_be_paid_remote_items=['42', '35'])
     mock_request.side_effect = [
         mock_json_item,  # get invoice 42
         mock_5xx,  # pay invoice 42
@@ -1275,10 +1281,10 @@ def test_transaction_retry_failure(mock_request, remote_regie):
     ]
     appconfig.update_transactions()
     transaction.refresh_from_db()
-    assert transaction.to_be_paid_remote_items == '42'  # retry for first one
+    assert transaction.to_be_paid_remote_items == ['42']  # retry for first one
 
     # err on get invoice
-    Transaction.objects.update(to_be_paid_remote_items='42,35')
+    Transaction.objects.update(to_be_paid_remote_items=['42', '35'])
     mock_request.side_effect = [
         mock_err,  # get invoice 42
         mock_json_item,  # get invoice 35
@@ -1286,10 +1292,10 @@ def test_transaction_retry_failure(mock_request, remote_regie):
     ]
     appconfig.update_transactions()
     transaction.refresh_from_db()
-    assert transaction.to_be_paid_remote_items == '42'  # retry for first one
+    assert transaction.to_be_paid_remote_items == ['42']  # retry for first one
 
     # err on pay invoice
-    Transaction.objects.update(to_be_paid_remote_items='42,35')
+    Transaction.objects.update(to_be_paid_remote_items=['42', '35'])
     mock_request.side_effect = [
         mock_json_item,  # get invoice 42
         mock_err,  # pay invoice 42
@@ -1298,20 +1304,20 @@ def test_transaction_retry_failure(mock_request, remote_regie):
     ]
     appconfig.update_transactions()
     transaction.refresh_from_db()
-    assert transaction.to_be_paid_remote_items == '42'  # retry for first one
+    assert transaction.to_be_paid_remote_items == ['42']  # retry for first one
 
     # 4xx on get invoice
-    Transaction.objects.update(to_be_paid_remote_items='42,35')
+    Transaction.objects.update(to_be_paid_remote_items=['42', '35'])
     mock_request.side_effect = [
         mock_4xx,  # get invoice 42
         mock_5xx,  # get invoice 35
     ]
     appconfig.update_transactions()
     transaction.refresh_from_db()
-    assert transaction.to_be_paid_remote_items == '35'  # retry for the second only
+    assert transaction.to_be_paid_remote_items == ['35']  # retry for the second only
 
     # 4xx on pay invoice
-    Transaction.objects.update(to_be_paid_remote_items='42,35')
+    Transaction.objects.update(to_be_paid_remote_items=['42', '35'])
     mock_request.side_effect = [
         mock_json_item,  # get invoice 42
         mock_4xx,  # get invoice 35
@@ -1319,7 +1325,7 @@ def test_transaction_retry_failure(mock_request, remote_regie):
     ]
     appconfig.update_transactions()
     transaction.refresh_from_db()
-    assert transaction.to_be_paid_remote_items == '35'  # retry for the second only
+    assert transaction.to_be_paid_remote_items == ['35']  # retry for the second only
 
 
 def test_transaction_validate(app, key, regie, user):

tests/test_lingo_remote_regie.py

@@ -144,7 +144,7 @@ def test_remote_regie_active_invoices_cell(mock_send, remote_regie):
 
     # set the second one as paid
     Transaction.objects.create(
-        regie=remote_regie, remote_items=INVOICES[1]['id'], status=eopayment.PAID, end_date=now()
+        regie=remote_regie, remote_items=[INVOICES[1]['id']], status=eopayment.PAID, end_date=now()
     )
     content = cell.render(context)
     assert 'F-2016-One' in content