Merge branch 'dev' into dependabot/pip/cryptography-44.0.1

.copilot/phases/pre_build.sh

@@ -3,23 +3,20 @@
 # Exit early if something goes wrong
 set -e
 
-git_clone_base_url="https://codestar-connections.eu-west-2.amazonaws.com/git-http/$AWS_ACCOUNT_ID/eu-west-2/$CODESTAR_CONNECTION_ID/uktrade"
+if [ -f "./.gitmodules" ]; then
+  echo ".gitmodules file exists. Modifying URLs..."
+    account_id=$(echo $CODESTAR_CONNECTION_ARN | cut -d':' -f5)
+    connection_id=$(echo $CODESTAR_CONNECTION_ARN | cut -d'/' -f2)
+    git_clone_base_url="https://codestar-connections.eu-west-2.amazonaws.com/git-http/$account_id/eu-west-2/$connection_id/uktrade"
 
-git config --global credential.helper '!aws codecommit credential-helper $@'
-git config --global credential.UseHttpPath true
+    git config --global credential.helper '!aws codecommit credential-helper $@'
+    git config --global credential.UseHttpPath true
 
-cat <<EOF > ./.gitmodules
-[submodule "lite-content"]
-path = lite_content
-url = $git_clone_base_url/lite-content.git
-branch = master
-[submodule "lite_routing"]
-path = lite_routing
-url = $git_clone_base_url/lite-routing.git
-branch = main
-[submodule "django_db_anonymiser"]
-path = django_db_anonymiser
-url = $git_clone_base_url/django-db-anonymiser.git
-EOF
+    sed -i "s|url = [email protected]:uktrade/\(.*\).git|url = $git_clone_base_url/\1.git|g" ./.gitmodules
 
-git submodule update --init --recursive
+
+    git submodule update --init --recursive
+
+else
+  echo ".gitmodules file does not exist. No URLs to update."
+fi

README.md

@@ -38,7 +38,7 @@ Service for handling backend calls in LITE.
     - `git submodule init`
     - `git submodule update`
 
-  - Ensure Docker Desktop (Docker daemon) is running
+  - Ensure Docker Desktop (Docker Daemon) is running
 
   - Build and start Docker images:
 

api/addresses/serializers.py

@@ -19,7 +19,7 @@ class AddressSerializer(serializers.ModelSerializer):
     )
     postcode = serializers.CharField(max_length=10, required=False, error_messages={"blank": Addresses.POSTCODE})
     city = serializers.CharField(max_length=50, required=False, error_messages={"blank": Addresses.CITY})
-    region = serializers.CharField(max_length=50, required=False, error_messages={"blank": Addresses.REGION})
+    region = serializers.CharField(max_length=50, required=False, allow_blank=True, allow_null=True)
     country = CountrySerializerField(required=False)
 
     def validate_postcode(self, value):

api/addresses/tests/test_address_validation.py

@@ -0,0 +1,109 @@
+from django.urls import reverse
+from rest_framework import status
+from random import randint
+from unittest import mock
+from api.core.authentication import EXPORTER_USER_TOKEN_HEADER
+from api.users.libraries.user_to_token import user_to_token
+from test_helpers.clients import DataTestClient
+from api.organisations.enums import OrganisationType
+from lite_content.lite_api.strings import Addresses
+
+
+class AddressSerializerPostcodeValidationTest(DataTestClient):
+    url = reverse("organisations:organisations")
+
+    @mock.patch("api.core.celery_tasks.NotificationsAPIClient.send_email_notification")
+    def test_required_address_fields(self, mock_gov_notification):
+        data = {
+            "name": "Lemonworld Co",
+            "type": OrganisationType.COMMERCIAL,
+            "eori_number": "GB123456789000",
+            "sic_number": "01110",
+            "vat_number": "GB123456789",
+            "registration_number": "98765432",
+            "site": {
+                "name": "Headquarters",
+                "address": {
+                    "address_line_1": "42 Industrial Estate",
+                    "address_line_2": "w3r",
+                    "region": "",
+                    "city": "St Albans",
+                    "postcode": "BT32 4PX",
+                },
+            },
+            "user": {"email": "[email protected]"},
+            "phone_number": "+441234567895",
+            "website": "",
+        }
+
+        data["registration_number"] = "".join([str(randint(0, 9)) for _ in range(8)])
+        response = self.client.post(
+            self.url, data, **{EXPORTER_USER_TOKEN_HEADER: user_to_token(self.exporter_user.baseuser_ptr)}
+        )
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+
+    @mock.patch("api.core.celery_tasks.NotificationsAPIClient.send_email_notification")
+    def test_valid(self, mock_gov_notification):
+        data = {
+            "name": "Lemonworld Co",
+            "type": OrganisationType.COMMERCIAL,
+            "eori_number": "GB123456789000",
+            "sic_number": "01110",
+            "vat_number": "GB123456789",
+            "registration_number": "98765432",
+            "site": {
+                "name": "Headquarters",
+                "address": {
+                    "address_line_1": "42 Industrial Estate",
+                    "address_line_2": "Queens Road",
+                    "region": "Hertfordshire",
+                    "city": "St Albans",
+                    "postcode": "BT32 4PX",
+                },
+            },
+            "user": {"email": "[email protected]"},
+            "phone_number": "+441234567895",
+            "website": "",
+        }
+
+        data["registration_number"] = "".join([str(randint(0, 9)) for _ in range(8)])
+        response = self.client.post(
+            self.url, data, **{EXPORTER_USER_TOKEN_HEADER: user_to_token(self.exporter_user.baseuser_ptr)}
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+
+    @mock.patch("api.core.celery_tasks.NotificationsAPIClient.send_email_notification")
+    def test_empty_address_fields(self, mock_gov_notification):
+        data = {
+            "name": "Lemonworld Co",
+            "type": OrganisationType.COMMERCIAL,
+            "eori_number": "GB123456789000",
+            "sic_number": "01110",
+            "vat_number": "GB123456789",
+            "registration_number": "98765432",
+            "site": {
+                "name": "Headquarters",
+                "address": {
+                    "address_line_1": "",
+                    "address_line_2": "",
+                    "region": "",
+                    "city": "",
+                    "postcode": "",
+                },
+            },
+            "user": {"email": "[email protected]"},
+            "phone_number": "+441234567895",
+            "website": "",
+        }
+
+        data["registration_number"] = "".join([str(randint(0, 9)) for _ in range(8)])
+        response = self.client.post(
+            self.url, data, **{EXPORTER_USER_TOKEN_HEADER: user_to_token(self.exporter_user.baseuser_ptr)}
+        )
+        address = response.json()["errors"]["site"]["address"]
+
+        self.assertEqual(address["postcode"][0], Addresses.POSTCODE)
+        self.assertEqual(address["address_line_1"][0], Addresses.ADDRESS_LINE_1)
+        self.assertEqual(address["city"][0], Addresses.CITY)
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)

api/data_workspace/urls.py

@@ -3,15 +3,11 @@
     path,
 )
 
-from api.data_workspace.v0.urls import router_v0
-from api.data_workspace.v1.urls import router_v1
-from api.data_workspace.v2.urls import router_v2
-
 
 app_name = "data_workspace"
 
 urlpatterns = [
-    path("v0/", include((router_v0.urls, "data_workspace_v0"), namespace="v0")),
-    path("v1/", include((router_v1.urls, "data_workspace_v1"), namespace="v1")),
-    path("v2/", include((router_v2.urls, "data_workspace_v2"), namespace="v2")),
+    path("v0/", include(("api.data_workspace.v0.urls", "data_workspace_v0"), namespace="v0")),
+    path("v1/", include(("api.data_workspace.v1.urls", "data_workspace_v1"), namespace="v1")),
+    path("v2/", include(("api.data_workspace.v2.urls", "data_workspace_v2"), namespace="v2")),
 ]

api/data_workspace/v0/urls.py

@@ -9,3 +9,5 @@
     licence_views.LicencesListDW,
     basename="dw-licences-only",
 )
+
+urlpatterns = router_v0.urls

api/data_workspace/v1/audit_views.py

@@ -1,4 +1,5 @@
 from django.contrib.contenttypes.models import ContentType
+from django.db import connection
 from rest_framework import viewsets
 from rest_framework.pagination import LimitOffsetPagination
 
@@ -15,22 +16,52 @@
 )
 
 
-class AuditMoveCaseListView(viewsets.ReadOnlyModelViewSet):
-    """Expose 'move case' audit events to data workspace."""
-
-    authentication_classes = (DataWorkspaceOnlyAuthentication,)
-    serializer_class = AuditMoveCaseSerializer
-    pagination_class = LimitOffsetPagination
-
-    def get_queryset(self):
+class AuditMoveCaseRawQuerySet:
+    # This is pretty horrible but it solves a problem in the "simplest way"
+    # Originally this was just Audit.objects.raw which then gets sent through
+    # DRFs gubbins to get a paginated queryset.
+    #
+    # The problem here is that by default the raw queryset is going to bring
+    # back the entire resultset to get both the count and the paginated results.
+    #
+    # In the case for this endpoint we're looping through 10,000s of rows,
+    # which was then transforming all of those rows into actual Audit objects
+    # putting them in a list and then doing len() and [0:25] on those lists.
+    #
+    # This was all hugely inefficient and wasteful.
+    #
+    # This horrible queryset allows us to just provide the paginator with what
+    # it needs quickly and efficiently.
+    #
+    # This could have been achieved by overriding a whole bunch in the APIView
+    # itself but that would have taken a lot more code and in this case we get
+    # to keep using what DRF wants to do internally.
+
+    def count(self):
         content_type = ContentType.objects.get_for_model(Case)
+        with connection.cursor() as cursor:
+            cursor.execute(
+                """
+                with audit_move_case as (
+                    select *,
+                    case when jsonb_typeof(payload->'queues') = 'array'
+                    then payload->'queues'
+                    else jsonb_build_array(payload->'queues')
+                    end as queues
+                    from audit_trail_audit
+                    where verb = %(verb)s
+                    and (action_object_content_type_id = %(action_type)s
+                    or target_content_type_id = %(target_type)s)
+                    order by created_at
+                )
+                select count(*) from audit_move_case cross join jsonb_array_elements(queues)""",
+                {"verb": AuditType.MOVE_CASE, "action_type": content_type.pk, "target_type": content_type.pk},
+            )
+            row = cursor.fetchone()
+        return row[0]
 
-        # This returns a queryset of audit records for the "move case" audit event.
-        # For each record, it exposes the nested "queues" JSON property as a top
-        # level column called "queue" and splits into multiple rows when "queues"
-        # contains multiple entries.
-        # It also deals with the fact that the value of "queues" is sometimes an
-        # array of queue names but sometimes a single string.
+    def __getitem__(self, slice):
+        content_type = ContentType.objects.get_for_model(Case)
         return Audit.objects.raw(
             """
             with audit_move_case as (
@@ -45,12 +76,27 @@ def get_queryset(self):
                 or target_content_type_id = %(target_type)s)
               order by created_at
             )
-            select *, value->>0 as "queue" from audit_move_case cross join jsonb_array_elements(queues)
+            select *, value->>0 as "queue" from audit_move_case cross join jsonb_array_elements(queues) LIMIT %(limit)s OFFSET %(offset)s
             """,
-            {"verb": AuditType.MOVE_CASE, "action_type": content_type.pk, "target_type": content_type.pk},
+            {
+                "verb": AuditType.MOVE_CASE,
+                "action_type": content_type.pk,
+                "target_type": content_type.pk,
+                "offset": slice.start,
+                "limit": slice.stop - slice.start,
+            },
         )
 
 
+class AuditMoveCaseListView(viewsets.ReadOnlyModelViewSet):
+    """Expose 'move case' audit events to data workspace."""
+
+    authentication_classes = (DataWorkspaceOnlyAuthentication,)
+    serializer_class = AuditMoveCaseSerializer
+    pagination_class = LimitOffsetPagination
+    queryset = AuditMoveCaseRawQuerySet()
+
+
 class AuditUpdatedCaseStatusListView(viewsets.ReadOnlyModelViewSet):
     """Expose 'updated status' audit events to data workspace."""
 

api/data_workspace/v1/urls.py

@@ -92,3 +92,5 @@
 router_v1.register("survey-response", views.SurveyResponseListView, basename="dw-survey-reponse")
 router_v1.register("address", address_views.AddressView, basename="dw-address")
 router_v1.register("site", organisations_views.SiteView, basename="dw-site")
+
+urlpatterns = router_v1.urls

api/data_workspace/v2/urls.py

@@ -13,3 +13,5 @@
 router_v2.register(views.FootnoteViewSet)
 router_v2.register(views.AssessmentViewSet)
 router_v2.register(views.LicenceRefusalCriteriaViewSet)
+
+urlpatterns = router_v2.urls

api/goods/serializers.py

@@ -817,6 +817,7 @@ class GoodSerializerInternal(serializers.Serializer):
     firearm_details = FirearmDetailsSerializer(allow_null=True, required=False)
     is_precedent = serializers.BooleanField(required=False, default=False)
     product_description = serializers.CharField()
+    has_declared_at_customs = serializers.BooleanField()
 
     def get_documents(self, instance):
         documents = instance.gooddocument_set.all()

api/goods/tests/factories.py

@@ -4,7 +4,6 @@
 from api.goods import models
 from api.goods.enums import ItemCategory, Component, MilitaryUse, FirearmGoodType, GoodPvGraded
 from api.organisations.tests.factories import OrganisationFactory
-from api.staticdata.control_list_entries.helpers import get_control_list_entry
 from api.staticdata.control_list_entries.models import ControlListEntry
 
 
@@ -58,6 +57,7 @@ class GoodFactory(factory.django.DjangoModelFactory):
     report_summary_subject = None
     report_summary = None
     comment = None
+    has_declared_at_customs = None
 
     class Meta:
         model = models.Good

api/goods/tests/test_serializers.py

@@ -182,6 +182,12 @@ def test_validate_good_internal_name_invalid(self, name, error_message):
             error_message,
         )
 
+    @parameterized.expand([None, True, False])
+    def test_has_declared_at_customs(self, has_declared_at_customs):
+        self.good.has_declared_at_customs = has_declared_at_customs
+        serialized_data = GoodSerializerInternal(self.good).data
+        self.assertEqual(serialized_data["has_declared_at_customs"], has_declared_at_customs)
+
 
 class GoodSerializerExporterFullDetailTests(DataTestClient):