From cdfd0e3f62ea1b4e642390ae006ac7fe71baa801 Mon Sep 17 00:00:00 2001 From: Wassilios Lytras Date: Mon, 16 Sep 2024 17:14:16 +0200 Subject: [PATCH 01/12] Add select_related to Admin models and Views to reduce number of queries. --- pyas2/admin.py | 6 ++++++ pyas2/models.py | 14 ++++++++++++-- pyas2/views.py | 14 +++++++++++--- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/pyas2/admin.py b/pyas2/admin.py index f9f1e37..4e2e9c9 100644 --- a/pyas2/admin.py +++ b/pyas2/admin.py @@ -169,6 +169,11 @@ class MessageAdmin(admin.ModelAdmin): "mdn_url", ] + list_select_related = ( + "partner", + "organization", + ) + @staticmethod def mdn_url(obj): """Return the URL to the related MDN if present for the message.""" @@ -212,6 +217,7 @@ class MdnAdmin(admin.ModelAdmin): ) list_display = ("mdn_id", "message", "timestamp", "status") list_filter = ("status",) + list_select_related = ("message",) def has_add_permission(self, request): return False diff --git a/pyas2/models.py b/pyas2/models.py index 7463cdf..8348d58 100644 --- a/pyas2/models.py +++ b/pyas2/models.py @@ -342,9 +342,12 @@ def create_from_as2message( if not filename: filename = f"{uuid4()}.msg" message.headers.save( - name=f"{filename}.header", content=ContentFile(as2message.headers_str) + name=f"{filename}.header", + content=ContentFile(as2message.headers_str), + save=False, ) - message.payload.save(name=filename, content=ContentFile(payload)) + message.payload.save(name=filename, content=ContentFile(payload), save=False) + message.save() # Save the payload to the inbox folder full_filename = None @@ -460,6 +463,13 @@ def status_icon(self): def send_message(self, header, payload): """Send the message to the partner""" + + if self.organization_id and self.partner_id: + self.organization = self.organization or Organization.objects.get( + id=self.organization_id + ) + self.partner = self.partner or Partner.objects.get(id=self.partner_id) + logger.info( f'Sending message {self.message_id} from organization "{self.organization}" ' f'to partner "{self.partner}".' diff --git a/pyas2/views.py b/pyas2/views.py index c205871..cdf38f0 100644 --- a/pyas2/views.py +++ b/pyas2/views.py @@ -56,7 +56,11 @@ def check_message_exists(message_id, partner_id): @staticmethod def find_organization(org_id): """Find the org using the As2 Id and return its pyas2 type""" - org = Organization.objects.filter(as2_name=org_id).first() + org = ( + Organization.objects.select_related("encryption_key", "signature_key") + .filter(as2_name=org_id) + .first() + ) if org: return org.as2org return None @@ -64,7 +68,11 @@ def find_organization(org_id): @staticmethod def find_partner(partner_id): """Find the partner using the As2 Id and return its pyas2 type""" - partner = Partner.objects.filter(as2_name=partner_id).first() + partner = ( + Partner.objects.select_related("encryption_cert", "signature_cert") + .filter(as2_name=partner_id) + .first() + ) if partner: return partner.as2partner return None @@ -97,7 +105,7 @@ def post(self, request, *args, **kwargs): status, detailed_status = as2mdn.parse(request_body, self.find_message) if not detailed_status == "mdn-not-found": - message = Message.objects.get( + message = Message.objects.select_related("organization", "partner").get( message_id=as2mdn.orig_message_id, direction="OUT" ) logger.info( From a1a92aa6b74c7094af213bf1c846ad4d0da26cca Mon Sep 17 00:00:00 2001 From: Wassilios Lytras Date: Mon, 16 Sep 2024 17:38:33 +0200 Subject: [PATCH 02/12] Add select_related to Admin models and Views to reduce number of queries. --- pyas2/tests/test_basic.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pyas2/tests/test_basic.py b/pyas2/tests/test_basic.py index 7cb259c..9c631f1 100644 --- a/pyas2/tests/test_basic.py +++ b/pyas2/tests/test_basic.py @@ -18,6 +18,8 @@ from pyas2lib.as2 import Message as As2Message +from django.test.utils import CaptureQueriesContext +from django.db import connection class BasicServerClientTestCase(TestCase): """Test cases for the AS2 server and client. @@ -534,6 +536,29 @@ def testEncryptSignMessageAsyncSignMdn(self, mock_request): mock_request.side_effect = RequestException() out_message.mdn.send_async_mdn() + def testNumberOfQueries(self): + """Testing against the number of queries executed""" + + # Create the partner with appropriate settings for this case + + partner = Partner.objects.create( + name="AS2 Server", + as2_name="as2server", + target_url="http://localhost:8080/pyas2/as2receive", + mdn=False, + ) + + with CaptureQueriesContext(connection) as queries: + in_message = self.build_and_send(partner) + + # Remove the transaction related queries + filtered_queries = [ + query for query in queries if 'SAVEPOINT' not in query['sql'] + ] + + # number of queries should be 9 + self.assertEqual(len(filtered_queries), 13) + @mock.patch("requests.post") def build_and_send(self, partner, mock_request): # Build and send the message to server From 195a687ad26321fb3c9852d9cd7573cb4f67225e Mon Sep 17 00:00:00 2001 From: Wassilios Lytras Date: Mon, 16 Sep 2024 17:40:40 +0200 Subject: [PATCH 03/12] Add select_related to Admin models and Views to reduce number of queries. --- pyas2/tests/test_basic.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyas2/tests/test_basic.py b/pyas2/tests/test_basic.py index 9c631f1..ffcbf65 100644 --- a/pyas2/tests/test_basic.py +++ b/pyas2/tests/test_basic.py @@ -21,6 +21,7 @@ from django.test.utils import CaptureQueriesContext from django.db import connection + class BasicServerClientTestCase(TestCase): """Test cases for the AS2 server and client. We will be testing each permutation as defined in RFC 4130 Section 2.4.2 @@ -553,7 +554,7 @@ def testNumberOfQueries(self): # Remove the transaction related queries filtered_queries = [ - query for query in queries if 'SAVEPOINT' not in query['sql'] + query for query in queries if "SAVEPOINT" not in query["sql"] ] # number of queries should be 9 From 64f4f2840a436de8089dc0ee559179e1da590dad Mon Sep 17 00:00:00 2001 From: Wassilios Lytras Date: Mon, 16 Sep 2024 17:58:17 +0200 Subject: [PATCH 04/12] Add select_related to Admin models and Views to reduce number of queries. --- pyas2/tests/test_basic.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/pyas2/tests/test_basic.py b/pyas2/tests/test_basic.py index ffcbf65..bcae408 100644 --- a/pyas2/tests/test_basic.py +++ b/pyas2/tests/test_basic.py @@ -2,25 +2,23 @@ from email.parser import HeaderParser from unittest import mock -from django.test import TestCase, Client +from django.db import connection +from django.test import Client, TestCase +from django.test.utils import CaptureQueriesContext +from pyas2lib.as2 import Message as As2Message from requests import Response from requests.exceptions import RequestException from pyas2.models import ( - PrivateKey, - PublicCertificate, + Mdn, + Message, Organization, Partner, - Message, - Mdn, + PrivateKey, + PublicCertificate, ) from pyas2.tests import TEST_DIR -from pyas2lib.as2 import Message as As2Message - -from django.test.utils import CaptureQueriesContext -from django.db import connection - class BasicServerClientTestCase(TestCase): """Test cases for the AS2 server and client. From 75fece16c36871a02b608cb9d98f1070e66790dc Mon Sep 17 00:00:00 2001 From: Wassilios Lytras Date: Tue, 17 Sep 2024 11:23:08 +0200 Subject: [PATCH 05/12] Add select_related to Partners and extend Message with mdn --- pyas2/admin.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyas2/admin.py b/pyas2/admin.py index 4e2e9c9..2bd293a 100644 --- a/pyas2/admin.py +++ b/pyas2/admin.py @@ -72,6 +72,10 @@ class PartnerAdmin(admin.ModelAdmin): "mdn_mode", ] list_filter = ("name", "as2_name") + list_select_related = ( + "encryption_cert", + "signature_cert", + ) fieldsets = ( ( None, @@ -172,6 +176,7 @@ class MessageAdmin(admin.ModelAdmin): list_select_related = ( "partner", "organization", + "mdn", ) @staticmethod From a621164717aa5e1e467fc3f1c5f4d516570fea62 Mon Sep 17 00:00:00 2001 From: Wassilios Lytras Date: Thu, 19 Sep 2024 11:38:11 +0200 Subject: [PATCH 06/12] Make creation of AS2 message in command atomic so that MDN received before command finishes does not fail and reduce queries on sending by 2. --- pyas2/management/commands/sendas2message.py | 19 +++++++++----- pyas2/tests/test_commands.py | 29 ++++++++++++++------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/pyas2/management/commands/sendas2message.py b/pyas2/management/commands/sendas2message.py index cd04c41..ead3976 100644 --- a/pyas2/management/commands/sendas2message.py +++ b/pyas2/management/commands/sendas2message.py @@ -4,6 +4,7 @@ from django.core.management.base import BaseCommand from django.core.management.base import CommandError from django.core.files.storage import default_storage +from django.db import transaction from pyas2lib import Message as AS2Message from pyas2.models import Message @@ -66,13 +67,17 @@ def handle(self, *args, **options): content_type=partner.content_type, disposition_notification_to=org.email_address or "no-reply@pyas2.com", ) - message, _ = Message.objects.create_from_as2message( - as2message=as2message, - payload=payload, - filename=original_filename, - direction="OUT", - status="P", - ) + + with transaction.atomic(): + message, _ = Message.objects.create_from_as2message( + as2message=as2message, + payload=payload, + filename=original_filename, + direction="OUT", + status="P", + ) + message.organization = org + message.partner = partner message.send_message(as2message.headers, as2message.content) # Delete original file if option is set diff --git a/pyas2/tests/test_commands.py b/pyas2/tests/test_commands.py index eb4006d..5301448 100644 --- a/pyas2/tests/test_commands.py +++ b/pyas2/tests/test_commands.py @@ -7,11 +7,13 @@ from django.conf import settings from django.core import management from django.core.files.base import ContentFile +from django.db import connection +from django.test.utils import CaptureQueriesContext from pyas2 import settings as app_settings -from pyas2.models import As2Message, Message, Mdn -from pyas2.tests import TEST_DIR from pyas2.management.commands.sendas2bulk import Command as SendBulkCommand +from pyas2.models import As2Message, Mdn, Message +from pyas2.tests import TEST_DIR @pytest.mark.django_db @@ -78,14 +80,23 @@ def test_sendmessage_command(mocker, organization, partner): mocked_delete = mocker.patch( "pyas2.management.commands.sendas2message.default_storage.delete" ) - management.call_command( - "sendas2message", - organization.as2_name, - partner.as2_name, - test_message, - delete=True, - ) + + with CaptureQueriesContext(connection) as queries: + + management.call_command( + "sendas2message", + organization.as2_name, + partner.as2_name, + test_message, + delete=True, + ) + + filtered_queries = [ + query for query in queries if "SAVEPOINT" not in query["sql"] + ] + assert mocked_delete.call_count == 1 + assert len(filtered_queries) == 6 @pytest.mark.django_db From 1ea08401f5dfc5cc78a36cec010c000be29fbd2f Mon Sep 17 00:00:00 2001 From: Wassilios Lytras Date: Tue, 24 Sep 2024 16:59:47 +0200 Subject: [PATCH 07/12] Adding instant transaction commit before message is sent. --- pyas2/management/commands/sendas2message.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/pyas2/management/commands/sendas2message.py b/pyas2/management/commands/sendas2message.py index ead3976..016e35c 100644 --- a/pyas2/management/commands/sendas2message.py +++ b/pyas2/management/commands/sendas2message.py @@ -68,14 +68,18 @@ def handle(self, *args, **options): disposition_notification_to=org.email_address or "no-reply@pyas2.com", ) - with transaction.atomic(): - message, _ = Message.objects.create_from_as2message( - as2message=as2message, - payload=payload, - filename=original_filename, - direction="OUT", - status="P", - ) + message, _ = Message.objects.create_from_as2message( + as2message=as2message, + payload=payload, + filename=original_filename, + direction="OUT", + status="P", + ) + + # Check if we're inside an atomic block, if not, commit immediately to store the message + if not transaction.get_connection().in_atomic_block: + transaction.commit() + message.organization = org message.partner = partner message.send_message(as2message.headers, as2message.content) From 1de80ce48cfd0c98fc2488a54a8208e429d44f63 Mon Sep 17 00:00:00 2001 From: Wassilios Lytras Date: Fri, 27 Sep 2024 08:31:36 +0200 Subject: [PATCH 08/12] Add/correct comments --- pyas2/management/commands/sendas2message.py | 4 +++- pyas2/tests/test_basic.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pyas2/management/commands/sendas2message.py b/pyas2/management/commands/sendas2message.py index 016e35c..61c8f36 100644 --- a/pyas2/management/commands/sendas2message.py +++ b/pyas2/management/commands/sendas2message.py @@ -76,8 +76,10 @@ def handle(self, *args, **options): status="P", ) - # Check if we're inside an atomic block, if not, commit immediately to store the message + # Check if we're inside an atomic block (f.e. in Django Test client). Commit to DB fails in such a case. if not transaction.get_connection().in_atomic_block: + # Store the message before sending. This prevents a "message not found" error when an async MDN is received + # from the partner before the sendas2message command as finished and is commited to DB. transaction.commit() message.organization = org diff --git a/pyas2/tests/test_basic.py b/pyas2/tests/test_basic.py index bcae408..3b20ead 100644 --- a/pyas2/tests/test_basic.py +++ b/pyas2/tests/test_basic.py @@ -555,7 +555,7 @@ def testNumberOfQueries(self): query for query in queries if "SAVEPOINT" not in query["sql"] ] - # number of queries should be 9 + # Number of queries should be 13 self.assertEqual(len(filtered_queries), 13) @mock.patch("requests.post") From 71cb08ea2364622a6310c530103e50287b003ea5 Mon Sep 17 00:00:00 2001 From: Wassilios Lytras Date: Fri, 27 Sep 2024 08:34:03 +0200 Subject: [PATCH 09/12] Add/correct comments --- pyas2/management/commands/sendas2message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyas2/management/commands/sendas2message.py b/pyas2/management/commands/sendas2message.py index 61c8f36..1138ee0 100644 --- a/pyas2/management/commands/sendas2message.py +++ b/pyas2/management/commands/sendas2message.py @@ -79,7 +79,7 @@ def handle(self, *args, **options): # Check if we're inside an atomic block (f.e. in Django Test client). Commit to DB fails in such a case. if not transaction.get_connection().in_atomic_block: # Store the message before sending. This prevents a "message not found" error when an async MDN is received - # from the partner before the sendas2message command as finished and is commited to DB. + # from the partner before the sendas2message command as finished and is commited to DB. transaction.commit() message.organization = org From 42dfc7c64aa4de8eaf58dfb25582dcc53f85b02b Mon Sep 17 00:00:00 2001 From: Wassilios Lytras Date: Fri, 27 Sep 2024 08:39:07 +0200 Subject: [PATCH 10/12] Add/correct comments --- pyas2/management/commands/sendas2message.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pyas2/management/commands/sendas2message.py b/pyas2/management/commands/sendas2message.py index 1138ee0..342589a 100644 --- a/pyas2/management/commands/sendas2message.py +++ b/pyas2/management/commands/sendas2message.py @@ -76,10 +76,12 @@ def handle(self, *args, **options): status="P", ) - # Check if we're inside an atomic block (f.e. in Django Test client). Commit to DB fails in such a case. + # Check if we're inside an atomic block (f.e. in Django Test client). Commit to DB fails + # in such a case. if not transaction.get_connection().in_atomic_block: - # Store the message before sending. This prevents a "message not found" error when an async MDN is received - # from the partner before the sendas2message command as finished and is commited to DB. + # Store the message before sending. This prevents a "message not found" error when an + # async MDN is received from the partner before the sendas2message command as finished + # and is commited to DB. transaction.commit() message.organization = org From c3287fa0dd01e435f6ded07423e8a6df9e6b8c8f Mon Sep 17 00:00:00 2001 From: Wassilios Lytras Date: Fri, 27 Sep 2024 10:45:55 +0200 Subject: [PATCH 11/12] Remove transaction commit when sending message. --- pyas2/management/commands/sendas2message.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pyas2/management/commands/sendas2message.py b/pyas2/management/commands/sendas2message.py index 342589a..16bbef8 100644 --- a/pyas2/management/commands/sendas2message.py +++ b/pyas2/management/commands/sendas2message.py @@ -76,14 +76,6 @@ def handle(self, *args, **options): status="P", ) - # Check if we're inside an atomic block (f.e. in Django Test client). Commit to DB fails - # in such a case. - if not transaction.get_connection().in_atomic_block: - # Store the message before sending. This prevents a "message not found" error when an - # async MDN is received from the partner before the sendas2message command as finished - # and is commited to DB. - transaction.commit() - message.organization = org message.partner = partner message.send_message(as2message.headers, as2message.content) From 62099a983ead13aae4d41f381825f78ee7f983a5 Mon Sep 17 00:00:00 2001 From: Wassilios Lytras Date: Fri, 27 Sep 2024 11:07:21 +0200 Subject: [PATCH 12/12] Remove transaction commit when sending message. --- pyas2/management/commands/sendas2message.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyas2/management/commands/sendas2message.py b/pyas2/management/commands/sendas2message.py index 16bbef8..165279d 100644 --- a/pyas2/management/commands/sendas2message.py +++ b/pyas2/management/commands/sendas2message.py @@ -4,7 +4,6 @@ from django.core.management.base import BaseCommand from django.core.management.base import CommandError from django.core.files.storage import default_storage -from django.db import transaction from pyas2lib import Message as AS2Message from pyas2.models import Message