From 7c66c923586c0008cff495477668c496b641fbd7 Mon Sep 17 00:00:00 2001
From: Matthew Somerville <matthew@mysociety.org>
Date: Tue, 19 Mar 2024 11:49:25 +0000
Subject: [PATCH 01/23] [Bromley] Add new update state for referral processing.

---
 perllib/Open311/Endpoint/Role/mySociety.pm                   | 4 ++++
 perllib/Open311/Endpoint/Service/Request/Update/mySociety.pm | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/perllib/Open311/Endpoint/Role/mySociety.pm b/perllib/Open311/Endpoint/Role/mySociety.pm
index 069b234ef..faeb1aefa 100644
--- a/perllib/Open311/Endpoint/Role/mySociety.pm
+++ b/perllib/Open311/Endpoint/Role/mySociety.pm
@@ -342,6 +342,8 @@ sub learn_additional_types {
             'cancelled',
             'reopen',
             'for_triage',
+            'referred_to_veolia_streets', # Bromley passthrough
+            'referred_to_lbb_streets', # Bromley passthrough
         )
     );
     $schema->learn_type( 'tag:wiki.open311.org,GeoReport_v2:rx/status_extended_upper',
@@ -360,6 +362,8 @@ sub learn_additional_types {
             'CANCELLED',
             'REOPEN',
             'FOR_TRIAGE',
+            'REFERRED_TO_VEOLIA_STREETS', # Bromley passthrough
+            'REFERRED_TO_LBB_STREETS', # Bromley passthrough
         )
     );
     $schema->learn_type( 'tag:wiki.open311.org,GeoReport_v2:rx/service_request_update',
diff --git a/perllib/Open311/Endpoint/Service/Request/Update/mySociety.pm b/perllib/Open311/Endpoint/Service/Request/Update/mySociety.pm
index de62b6165..388c5b7a9 100644
--- a/perllib/Open311/Endpoint/Service/Request/Update/mySociety.pm
+++ b/perllib/Open311/Endpoint/Service/Request/Update/mySociety.pm
@@ -22,6 +22,8 @@ has status => (
         'cancelled',
         'reopen',
         'for_triage',
+        'referred_to_veolia_streets', # Bromley passthrough
+        'referred_to_lbb_streets', # Bromley passthrough
     ],
 );
 

From 5a5b50859e36872253bbbfca22102f249b0bd314 Mon Sep 17 00:00:00 2001
From: Matthew Somerville <matthew@mysociety.org>
Date: Fri, 22 Mar 2024 15:05:06 +0000
Subject: [PATCH 02/23] [Bromley] Include Title, and single-line Notes.

---
 .../Endpoint/Integration/UK/Bromley/Echo.pm   | 20 +++++++++++++++++++
 t/open311/endpoint/bromley_echo.t             | 19 ++++++++++++++----
 2 files changed, 35 insertions(+), 4 deletions(-)

diff --git a/perllib/Open311/Endpoint/Integration/UK/Bromley/Echo.pm b/perllib/Open311/Endpoint/Integration/UK/Bromley/Echo.pm
index 2bb7f2f84..beedc53fe 100644
--- a/perllib/Open311/Endpoint/Integration/UK/Bromley/Echo.pm
+++ b/perllib/Open311/Endpoint/Integration/UK/Bromley/Echo.pm
@@ -30,9 +30,29 @@ start/end/review dates; when removing, we set the action and end date.
 
 =cut
 
+sub echo_title_id {
+    my $title = shift;
+    return 1 if $title eq 'MR';
+    return 2 if $title eq 'MRS';
+    return 3 if $title eq 'MISS';
+    return 4 if $title eq 'MS';
+    return 5 if $title eq 'DR';
+    return 7;
+}
+
 around process_service_request_args => sub {
     my ($orig, $class, $args) = @_;
     my $request = $class->$orig($args);
+
+    if (!$args->{attributes}{Notes}) {
+        $args->{attributes}{Notes} = join(" | ", $args->{attributes}{report_title} || (), $args->{description} || ());
+    }
+    $args->{attributes}{Notes} =~ s/(\r?\n)+/ | /g;
+
+    if (my $title = $args->{attributes}{title}) {
+        $args->{attributes}{Title} = echo_title_id($title);
+    }
+
     # Assisted collection
     if ($request->{event_type} == 2149) {
         my $date = DateTime->today(time_zone => "Europe/London");
diff --git a/t/open311/endpoint/bromley_echo.t b/t/open311/endpoint/bromley_echo.t
index 3a0ff52df..4389012e6 100644
--- a/t/open311/endpoint/bromley_echo.t
+++ b/t/open311/endpoint/bromley_echo.t
@@ -62,8 +62,8 @@ $soap_lite->mock(call => sub {
 
         my @data = ${$params[0]->value}->value->value;
         is $uprn, 1000001;
-        if (@data == 6) {
-            is @data, 6, 'Various supplied data';
+        if (@data == 8) {
+            is @data, 8, 'Various supplied data';
             my @action = ${$data[0]->value}->value;
             is $action[0]->value, 1001;
             is $action[1]->value, 1;
@@ -79,17 +79,26 @@ $soap_lite->mock(call => sub {
             my @loc = ${$data[4]->value}->value;
             is $loc[0]->value, 1005;
             is $loc[1]->value, 'Behind the wall';
-            my @type = ${$data[5]->value}->value;
+            my @notes = ${$data[5]->value}->value;
+            is $notes[0]->value, 1006;
+            is $notes[1]->value, 'This is the details';
+            my @type = ${$data[6]->value}->value;
             is $type[0]->value, 1007;
             is $type[1]->value, 3;
+            my @title = ${$data[7]->value}->value;
+            is $title[0]->value, 1008;
+            is $title[1]->value, 2;
         } else {
-            is @data, 2, 'Various supplied data';
+            is @data, 3, 'Various supplied data';
             my @action = ${$data[0]->value}->value;
             is $action[0]->value, 1001;
             is $action[1]->value, 2;
             my @start = ${$data[1]->value}->value;
             is $start[0]->value, 1003;
             is $start[1]->value, $date->strftime("%d/%m/%Y");
+            my @notes = ${$data[2]->value}->value;
+            is $notes[0]->value, 1006;
+            is $notes[1]->value, 'This is the details';
         }
 
         return SOAP::Result->new(result => {
@@ -107,6 +116,7 @@ $soap_lite->mock(call => sub {
                 { Id => 1005, Name => "Exact Location" },
                 { Id => 1006, Name => "Notes" },
                 { Id => 1007, Name => "Container Type" },
+                { Id => 1008, Name => "Title" },
             ] },
         });
     } elsif ($method eq 'PerformEventAction') {
@@ -142,6 +152,7 @@ subtest "POST add assisted collection OK" => sub {
         'attribute[fixmystreet_id]' => 2000123,
         'attribute[Exact_Location]' => 'Behind the wall',
         'attribute[Container_Type]' => 3,
+        'attribute[title]' => 'MRS',
     );
     ok $res->is_success, 'valid request'
         or diag $res->content;

From 7732df96483bdab59508c7e946e02ef037d5fcae Mon Sep 17 00:00:00 2001
From: Matthew Somerville <matthew@mysociety.org>
Date: Fri, 22 Mar 2024 16:03:37 +0000
Subject: [PATCH 03/23] [Bromley] Add default Event ID.

---
 perllib/Open311/Endpoint/Integration/UK/Bromley/Echo.pm | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/perllib/Open311/Endpoint/Integration/UK/Bromley/Echo.pm b/perllib/Open311/Endpoint/Integration/UK/Bromley/Echo.pm
index beedc53fe..f2bd84ae8 100644
--- a/perllib/Open311/Endpoint/Integration/UK/Bromley/Echo.pm
+++ b/perllib/Open311/Endpoint/Integration/UK/Bromley/Echo.pm
@@ -53,6 +53,10 @@ around process_service_request_args => sub {
         $args->{attributes}{Title} = echo_title_id($title);
     }
 
+    if ($request->{event_type} == 3045 && !$args->{attributes}{Event_ID}) {
+        $args->{attributes}{Event_ID} = 'LBB_REFERRAL';
+    }
+
     # Assisted collection
     if ($request->{event_type} == 2149) {
         my $date = DateTime->today(time_zone => "Europe/London");

From ce9ac7d92f1681171e3c300c5a9af5a8596a9cea Mon Sep 17 00:00:00 2001
From: Moray Jones <moray@mysociety.org>
Date: Fri, 12 Apr 2024 11:11:28 +0100
Subject: [PATCH 04/23] [Bromley][WW] Truncate text to maximum field length

Notes text should not exceed maximum length for notes
field.

https://github.com/mysociety/societyworks/issues/4232
---
 .../Endpoint/Integration/UK/Bromley/Echo.pm   | 17 ++++++++++
 t/open311/endpoint/bromley_echo.t             | 32 ++++++++++++++++++-
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/perllib/Open311/Endpoint/Integration/UK/Bromley/Echo.pm b/perllib/Open311/Endpoint/Integration/UK/Bromley/Echo.pm
index f2bd84ae8..9c19e8ca4 100644
--- a/perllib/Open311/Endpoint/Integration/UK/Bromley/Echo.pm
+++ b/perllib/Open311/Endpoint/Integration/UK/Bromley/Echo.pm
@@ -49,6 +49,10 @@ around process_service_request_args => sub {
     }
     $args->{attributes}{Notes} =~ s/(\r?\n)+/ | /g;
 
+    if ($args->{attributes}{Notes} && $args->{attributes}{Notes} =~ m/^Closed report has a new comment:/ ) {
+        $args->{attributes}{Notes} = _truncate_text($args->{attributes}{Notes}, 2000, '...');
+    }
+
     if (my $title = $args->{attributes}{title}) {
         $args->{attributes}{Title} = echo_title_id($title);
     }
@@ -89,5 +93,18 @@ around post_service_request_update => sub {
     return $class->$orig($args);
 };
 
+sub _truncate_text {
+    my ($text, $max_length, $postscript) = @_;
+
+    return $text if length($text) <= $max_length;
+
+    if ($postscript) {
+        $max_length = $max_length - length($postscript);
+    };
+
+    return substr( $text, 0, $max_length ) . $postscript;
+}
+
+
 1;
 
diff --git a/t/open311/endpoint/bromley_echo.t b/t/open311/endpoint/bromley_echo.t
index 4389012e6..58b225870 100644
--- a/t/open311/endpoint/bromley_echo.t
+++ b/t/open311/endpoint/bromley_echo.t
@@ -81,7 +81,11 @@ $soap_lite->mock(call => sub {
             is $loc[1]->value, 'Behind the wall';
             my @notes = ${$data[5]->value}->value;
             is $notes[0]->value, 1006;
-            is $notes[1]->value, 'This is the details';
+            if ($notes[1]->value =~ /^Closed report has a/) {
+                is $notes[1]->value, 'Closed report has a new comment: comment that was left | Jay User user@user.com | This was the text!' . 'a' x 1897 . '...',
+            } else {
+                is $notes[1]->value, 'This is the details';
+            }
             my @type = ${$data[6]->value}->value;
             is $type[0]->value, 1007;
             is $type[1]->value, 3;
@@ -163,6 +167,32 @@ subtest "POST add assisted collection OK" => sub {
         } ], 'correct json returned';
 };
 
+subtest "POST add assisted collection OK" => sub {
+    my $res = $endpoint->run_test_request(
+        POST => '/requests.json',
+        api_key => 'test',
+        service_code => EVENT_TYPE_ASSISTED . "-add",
+        first_name => 'Bob',
+        last_name => 'Mould',
+        description => "Closed report has a new comment: comment that was left\r\nJay User user\@user.com\r\nThis was the text!" . 'a' x 2000,
+        lat => 51,
+        long => -1,
+        'attribute[service_id]' => 531, # Domestic refuse
+        'attribute[uprn]' => 1000001,
+        'attribute[fixmystreet_id]' => 2000123,
+        'attribute[Exact_Location]' => 'Behind the wall',
+        'attribute[Container_Type]' => 3,
+        'attribute[title]' => 'MRS',
+    );
+    ok $res->is_success, 'valid request'
+        or diag $res->content;
+
+    is_deeply decode_json($res->content),
+        [ {
+            "service_request_id" => '1234',
+        } ], 'correct json returned';
+};
+
 subtest "POST remove assisted collection OK" => sub {
     my $res = $endpoint->run_test_request(
         POST => '/requests.json',

From e1b7c5d4ad5883f5569dc5147f836b1f1e4f589b Mon Sep 17 00:00:00 2001
From: Matthew Somerville <matthew@mysociety.org>
Date: Wed, 31 Jul 2024 13:52:10 +0100
Subject: [PATCH 05/23] Allow through attribute[]s on any updates.

---
 perllib/Open311/Endpoint/Role/mySociety.pm | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/perllib/Open311/Endpoint/Role/mySociety.pm b/perllib/Open311/Endpoint/Role/mySociety.pm
index faeb1aefa..8cc9c8747 100644
--- a/perllib/Open311/Endpoint/Role/mySociety.pm
+++ b/perllib/Open311/Endpoint/Role/mySociety.pm
@@ -196,11 +196,9 @@ sub POST_Service_Request_Update_input_schema {
         delete $attributes->{required}{update_id};
     }
 
-    # Allow attributes through for Oxfordshire XXX
-    if ($jurisdiction eq 'oxfordshire') {
-        for my $key (grep { /^attribute\[\w+\]$/ } keys %$args) {
-            $attributes->{optional}{$key} = '//str';
-        }
+    # Allow attributes through
+    for my $key (grep { /^attribute\[\w+\]$/ } keys %$args) {
+        $attributes->{optional}{$key} = '//str';
     }
 
     # Allow nsg_ref through for Bexley XXX

From 6252eb4da1282e8eb0f7869bc4f3b19951d0c3b0 Mon Sep 17 00:00:00 2001
From: Dave Arter <davea@mysociety.org>
Date: Tue, 15 Oct 2024 10:45:51 +0100
Subject: [PATCH 06/23] [Confirm] Allow ignoring of enquiries in certain status

Fetched Enquiries will be skipped if their status
maps to IGNORE in reverse_status_mapping.

Matches behaviour of update fetching in 42f75f6b.
---
 perllib/Open311/Endpoint/Integration/Confirm.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/perllib/Open311/Endpoint/Integration/Confirm.pm b/perllib/Open311/Endpoint/Integration/Confirm.pm
index 83fca8e4a..78beaf522 100644
--- a/perllib/Open311/Endpoint/Integration/Confirm.pm
+++ b/perllib/Open311/Endpoint/Integration/Confirm.pm
@@ -825,6 +825,7 @@ sub get_service_requests {
         my $code = $enquiry->{ServiceCode} . "_" . $enquiry->{SubjectCode};
         my $service = $services{$code};
         my $status = $self->reverse_status_mapping->{$enquiry->{EnquiryStatusCode}};
+        next if $status && $status eq 'IGNORE';
 
         unless ($service || ($service = $self->_find_wrapping_service($code, \@services))) {
             $self->logger->warn("no service for service code $code");

From 3510dedf2c9d415a08fec201118d2e82bcf4585d Mon Sep 17 00:00:00 2001
From: Matthew Somerville <matthew@mysociety.org>
Date: Tue, 15 Oct 2024 11:14:09 +0100
Subject: [PATCH 07/23] [Bexley] Rename Symology service class.

---
 perllib/Open311/Endpoint/Integration/UK/Bexley/Symology.pm    | 4 ++--
 .../UKCouncil/{BexleySymology.pm => Symology/Bexley.pm}       | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)
 rename perllib/Open311/Endpoint/Service/UKCouncil/{BexleySymology.pm => Symology/Bexley.pm} (85%)

diff --git a/perllib/Open311/Endpoint/Integration/UK/Bexley/Symology.pm b/perllib/Open311/Endpoint/Integration/UK/Bexley/Symology.pm
index f39072cc9..72a4ad275 100644
--- a/perllib/Open311/Endpoint/Integration/UK/Bexley/Symology.pm
+++ b/perllib/Open311/Endpoint/Integration/UK/Bexley/Symology.pm
@@ -5,7 +5,7 @@ package Open311::Endpoint::Integration::UK::Bexley::Symology;
 use Moo;
 extends 'Open311::Endpoint::Integration::Symology';
 
-use Open311::Endpoint::Service::UKCouncil::BexleySymology;
+use Open311::Endpoint::Service::UKCouncil::Symology::Bexley;
 
 has jurisdiction_id => (
     is => 'ro',
@@ -14,7 +14,7 @@ has jurisdiction_id => (
 
 has service_class  => (
     is => 'ro',
-    default => 'Open311::Endpoint::Service::UKCouncil::BexleySymology'
+    default => 'Open311::Endpoint::Service::UKCouncil::Symology::Bexley'
 );
 
 sub process_service_request_args {
diff --git a/perllib/Open311/Endpoint/Service/UKCouncil/BexleySymology.pm b/perllib/Open311/Endpoint/Service/UKCouncil/Symology/Bexley.pm
similarity index 85%
rename from perllib/Open311/Endpoint/Service/UKCouncil/BexleySymology.pm
rename to perllib/Open311/Endpoint/Service/UKCouncil/Symology/Bexley.pm
index d7dff1da2..cbae0c2a3 100644
--- a/perllib/Open311/Endpoint/Service/UKCouncil/BexleySymology.pm
+++ b/perllib/Open311/Endpoint/Service/UKCouncil/Symology/Bexley.pm
@@ -1,4 +1,4 @@
-package Open311::Endpoint::Service::UKCouncil::BexleySymology;
+package Open311::Endpoint::Service::UKCouncil::Symology::Bexley;
 
 use Moo;
 extends 'Open311::Endpoint::Service::UKCouncil::Symology';

From 38259c15ed07a53e566515b49a0eab83e82707b6 Mon Sep 17 00:00:00 2001
From: Matthew Somerville <matthew@mysociety.org>
Date: Tue, 15 Oct 2024 11:55:59 +0100
Subject: [PATCH 08/23] [Symology] Factor out URL/photo line adding.

---
 perllib/Integrations/Symology.pm              |  8 +---
 .../Open311/Endpoint/Integration/Symology.pm  | 38 ++++++++++++++++---
 .../Integration/UK/Bexley/Symology.pm         |  9 +++++
 .../Endpoint/Integration/UK/Brent/Symology.pm | 12 ------
 .../Integration/UK/Camden/Symology.pm         | 26 -------------
 .../UK/CentralBedfordshire/Symology.pm        | 10 -----
 t/open311/endpoint/brent_symology.yml         |  5 +++
 t/open311/endpoint/camden_symology.yml        |  3 ++
 .../endpoint/centralbedfordshire_symology.t   |  4 ++
 9 files changed, 54 insertions(+), 61 deletions(-)

diff --git a/perllib/Integrations/Symology.pm b/perllib/Integrations/Symology.pm
index 0827de0f9..c8880860f 100644
--- a/perllib/Integrations/Symology.pm
+++ b/perllib/Integrations/Symology.pm
@@ -187,13 +187,7 @@ sub current_date {
 sub SOAP::Serializer::as_ArrayOfAdditionalFieldSend {
     my ($self, $value, $name, $type, $attr) = @_;
 
-    # Allow individual integrations to provide their own values here, otherwise
-    # assume it's a simple value for the customer type field.
-    unless (ref $value eq 'ARRAY') {
-        $value = [
-            [ FieldLine => 17, ValueType => 1, DataValue => $value ],
-        ];
-    }
+    die unless ref $value eq 'ARRAY';
 
     my $elem = \SOAP::Data
         ->name('tns:AdditionalFieldSend' => map { [ make_soap_structure(@$_) ] } @$value)
diff --git a/perllib/Open311/Endpoint/Integration/Symology.pm b/perllib/Open311/Endpoint/Integration/Symology.pm
index 3b7541f1c..7859703fc 100644
--- a/perllib/Open311/Endpoint/Integration/Symology.pm
+++ b/perllib/Open311/Endpoint/Integration/Symology.pm
@@ -52,11 +52,28 @@ has update_urls => (
     default => sub { $_[0]->endpoint_config->{update_urls} }
 );
 
+=item * field_defaults contains info for use when constructing the additional fields
+
+=cut
+
+has field_defaults => (
+    is => 'lazy',
+    default => sub { $_[0]->endpoint_config->{field_defaults} || {} }
+);
+
+=item * customer_defaults contains info for use when constructing the customer
+
+=cut
+
 has customer_defaults => (
     is => 'lazy',
     default => sub { $_[0]->endpoint_config->{customer_defaults} }
 );
 
+=item * request_defaults contains info for use when constructing the request
+
+=cut
+
 has request_defaults => (
     is => 'lazy',
     default => sub { $_[0]->endpoint_config->{request_defaults} || {} }
@@ -179,19 +196,28 @@ sub process_service_request_args {
         }
     }
 
-    # Bit Bexley-specific still
-    my $contact_type = $self->customer_defaults->{ContactType};
-    $contact_type //= $request->{contributed_by} ? 'TL' : 'OL';
-
     my $customer = {
         name => $args->{first_name} . " " . $args->{last_name},
         email => $args->{email},
         phone => $args->{phone},
         customer_type => $self->customer_defaults->{CustomerType},
-        contact_type => $contact_type,
+        contact_type => $self->customer_defaults->{ContactType},
     };
 
-    my $fields = delete $request->{contributed_by};
+    my $fields = [];
+    if (my $field_line_value = $self->field_defaults->{URL}) {
+        push @$fields, [ FieldLine => $field_line_value, ValueType => 8, DataValue => $args->{attributes}->{report_url} ];
+    }
+    if (my $field_line_value = $self->field_defaults->{PhotoStart}) {
+        my $value_type = $self->field_defaults->{PhotoType} || 8;
+        foreach my $photo_url ( @{ $args->{media_url} } ) {
+            push @$fields, [ FieldLine => $field_line_value, ValueType => $value_type, DataValue => $photo_url ];
+
+            # Only send the first three photos
+            last if $field_line_value == $self->field_defaults->{PhotoEnd};
+            $field_line_value++;
+        }
+    }
 
     return ($request, $customer, $fields);
 }
diff --git a/perllib/Open311/Endpoint/Integration/UK/Bexley/Symology.pm b/perllib/Open311/Endpoint/Integration/UK/Bexley/Symology.pm
index 72a4ad275..0861595cd 100644
--- a/perllib/Open311/Endpoint/Integration/UK/Bexley/Symology.pm
+++ b/perllib/Open311/Endpoint/Integration/UK/Bexley/Symology.pm
@@ -21,9 +21,18 @@ sub process_service_request_args {
     my $self = shift;
     my @args = $self->SUPER::process_service_request_args(@_);
     my $request = $args[0];
+    my $customer = $args[1];
 
     $request->{NextAction} = $self->post_add_next_action_update($request->{NSGRef});
 
+    $customer->{contact_type} = $request->{contributed_by} ? 'TL' : 'OL';
+
+    push @{ $args[2] }, [
+        FieldLine => 17,
+        ValueType => 1,
+        DataValue => delete $request->{contributed_by},
+    ];
+
     return @args;
 }
 
diff --git a/perllib/Open311/Endpoint/Integration/UK/Brent/Symology.pm b/perllib/Open311/Endpoint/Integration/UK/Brent/Symology.pm
index b5f9adb48..d2a79b0dc 100644
--- a/perllib/Open311/Endpoint/Integration/UK/Brent/Symology.pm
+++ b/perllib/Open311/Endpoint/Integration/UK/Brent/Symology.pm
@@ -23,18 +23,6 @@ sub process_service_request_args {
     my $response = $args[0];
     $response->{Location} = $location;
 
-    push @{ $args[2] }, [ FieldLine => 3, ValueType => 8,  DataValue => $_[0]->{attributes}->{report_url} ];
-    # Add the photo URLs to the request
-    my $field_line_value = 4;
-
-    foreach my $photo_url ( @{ $_[0]->{media_url} } ) {
-        push @{ $args[2] }, [ FieldLine => $field_line_value, ValueType => 7, DataValue => $photo_url ];
-
-        # Only send the first three photos
-        last if $field_line_value == 6;
-
-        $field_line_value++;
-    };
     return @args;
 }
 
diff --git a/perllib/Open311/Endpoint/Integration/UK/Camden/Symology.pm b/perllib/Open311/Endpoint/Integration/UK/Camden/Symology.pm
index c4517b53f..fb0be9409 100644
--- a/perllib/Open311/Endpoint/Integration/UK/Camden/Symology.pm
+++ b/perllib/Open311/Endpoint/Integration/UK/Camden/Symology.pm
@@ -12,30 +12,4 @@ has jurisdiction_id => (
     default => 'camden_symology',
 );
 
-=head2 process_service_request_args
-
-Camden's Symology has got three fields for photo URLs. We send the first
-three photo URLs to those fields when creating a report.
-
-=cut
-
-sub process_service_request_args {
-    my $self = shift;
-
-    my @args = $self->SUPER::process_service_request_args(@_);
-
-    # Add the photo URLs to the request
-    my $field_line_value = 10;
-    foreach my $photo_url ( @{ $_[0]->{media_url} } ) {
-        push @{ $args[2] }, [ FieldLine => $field_line_value, ValueType => 8, DataValue => $photo_url ];
-
-        # Only send the first three photos
-        last if $field_line_value == 12;
-
-        $field_line_value++;
-    }
-
-    return @args;
-}
-
 1;
diff --git a/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Symology.pm b/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Symology.pm
index 0b64beaa5..d44f21a6e 100644
--- a/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Symology.pm
+++ b/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Symology.pm
@@ -33,16 +33,6 @@ sub process_service_request_args {
 
     $response->{Location} = $location;
 
-    # Send the first photo to the appropriate field in Symology
-    # (these values are specific to Central Beds and were divined by
-    # inspecting the GetRequestAdditionalGroup output for an existing
-    # enquiry that had a photo.)
-    if ( my $photo_url = $_[0]->{media_url}->[0] ) {
-        $args[2] = [
-            [ FieldLine => 15, ValueType => 8, DataValue => $photo_url ],
-        ];
-    }
-
     return @args;
 }
 
diff --git a/t/open311/endpoint/brent_symology.yml b/t/open311/endpoint/brent_symology.yml
index f33568961..d91939c5b 100644
--- a/t/open311/endpoint/brent_symology.yml
+++ b/t/open311/endpoint/brent_symology.yml
@@ -7,6 +7,11 @@ request_defaults:
     ServiceCode: 'ServiceCode'
 customer_defaults:
     CustomerType: "PB"
+field_defaults:
+  URL: 3
+  PhotoStart: 4
+  PhotoEnd: 6
+  PhotoType: 7
 event_action_event_type: CHRR
 event_status_mapping:
   19: 'investigating'
diff --git a/t/open311/endpoint/camden_symology.yml b/t/open311/endpoint/camden_symology.yml
index 3d9c4c350..2979ea192 100644
--- a/t/open311/endpoint/camden_symology.yml
+++ b/t/open311/endpoint/camden_symology.yml
@@ -2,6 +2,9 @@ username: FMS
 customer_defaults:
   CustomerType: ""
   ContactType: ""
+field_defaults:
+  PhotoStart: 10
+  PhotoEnd: 12
 event_action_event_type: NOTE
 category_mapping:
   Potholes:
diff --git a/t/open311/endpoint/centralbedfordshire_symology.t b/t/open311/endpoint/centralbedfordshire_symology.t
index be6f71bde..20b84af53 100644
--- a/t/open311/endpoint/centralbedfordshire_symology.t
+++ b/t/open311/endpoint/centralbedfordshire_symology.t
@@ -133,6 +133,10 @@ $centralbeds_symology_end->mock(endpoint_config => sub {
     {
         username => 'FMS',
         nsgref_to_action => {},
+        field_defaults => {
+            PhotoStart => 15,
+            PhotoEnd => 15,
+        },
         customer_defaults => {
             CustomerType => "",
             ContactType => "",

From 0695f1b73b70237e6229e8f02b12988f8b5b318c Mon Sep 17 00:00:00 2001
From: Matthew Somerville <matthew@mysociety.org>
Date: Wed, 16 Oct 2024 12:21:58 +0100
Subject: [PATCH 09/23] [Alloy] Log GET calls.

---
 perllib/Integrations/AlloyV2.pm | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/perllib/Integrations/AlloyV2.pm b/perllib/Integrations/AlloyV2.pm
index 8a20d5425..05fb9fcec 100644
--- a/perllib/Integrations/AlloyV2.pm
+++ b/perllib/Integrations/AlloyV2.pm
@@ -56,6 +56,8 @@ sub api_call {
         $request->content(encode_json($body));
         $self->logger->debug($call);
         $self->logger->debug(encode_json($body));
+    } else {
+        $self->logger->debug($call);
     }
     my $response = $ua->request($request);
     if ($response->is_success) {

From 97d6e2706fd34ccc4f6a60deeedb8f8c79b5a5af Mon Sep 17 00:00:00 2001
From: Matthew Somerville <matthew@mysociety.org>
Date: Tue, 29 Oct 2024 10:37:36 +0000
Subject: [PATCH 10/23] Add script to check endpoint certificate expiry dates.

---
 bin/check-endpoints                           | 24 +++++++++++++
 perllib/Integrations/Surrey/Boomi.pm          |  4 ++-
 perllib/Open311/Endpoint/Integration/Boomi.pm |  2 +-
 perllib/Open311/Endpoint/Integration/UK.pm    | 36 +++++++++++++++++++
 t/open311/endpoint/uk.t                       |  1 +
 5 files changed, 65 insertions(+), 2 deletions(-)
 create mode 100755 bin/check-endpoints

diff --git a/bin/check-endpoints b/bin/check-endpoints
new file mode 100755
index 000000000..389addda8
--- /dev/null
+++ b/bin/check-endpoints
@@ -0,0 +1,24 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+use v5.14;
+
+BEGIN {
+    use File::Basename qw(dirname);
+    use File::Spec;
+    my $d = dirname(File::Spec->rel2abs($0));
+    require "$d/../setenv.pl";
+}
+
+use Getopt::Long;
+use Open311::Endpoint::Integration::UK;
+
+my ($verbose);
+GetOptions (
+    'verbose|v' => \$verbose,
+);
+
+my $uk = Open311::Endpoint::Integration::UK->new;
+
+$uk->check_endpoints($verbose);
diff --git a/perllib/Integrations/Surrey/Boomi.pm b/perllib/Integrations/Surrey/Boomi.pm
index a25307c55..80fa55c07 100644
--- a/perllib/Integrations/Surrey/Boomi.pm
+++ b/perllib/Integrations/Surrey/Boomi.pm
@@ -38,7 +38,9 @@ has ua => (
             timeout => 3 * 60, # Boomi can take longer than the default 60s when handling large/multiple photos.
         );
         $ua->ssl_opts(SSL_cipher_list => 'DEFAULT:!DH'); # Disable DH ciphers, server's key is too small apparently
-        my $hash = encode_base64($self->config->{username} . ':' . $self->config->{password}, "");
+        my $un = $self->config->{username} || '';
+        my $pw = $self->config->{password} || '';
+        my $hash = encode_base64("$un:$pw", "");
         $ua->default_header('Authorization' => "Basic $hash");
         $ua->default_header('Content-Type' => "application/json");
         return $ua;
diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
index b4562cdeb..50323bf4d 100644
--- a/perllib/Open311/Endpoint/Integration/Boomi.pm
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -44,7 +44,7 @@ has integration_class => (
     default => 'Integrations::Surrey::Boomi',
 );
 
-
+sub get_integration { $_[0]->boomi }
 
 sub service {
     my ($self, $id, $args) = @_;
diff --git a/perllib/Open311/Endpoint/Integration/UK.pm b/perllib/Open311/Endpoint/Integration/UK.pm
index dccf1cf2c..d818646b1 100644
--- a/perllib/Open311/Endpoint/Integration/UK.pm
+++ b/perllib/Open311/Endpoint/Integration/UK.pm
@@ -12,6 +12,7 @@ use Module::Pluggable
     instantiate => 'new';
 use JSON::MaybeXS;
 use Path::Tiny;
+use URI;
 
 use Open311::Endpoint::Schema;
 
@@ -141,4 +142,39 @@ sub confirm_upload {
     }
 }
 
+sub check_endpoints {
+    my ($self, $verbose) = @_;
+    my @plugins = ();
+    foreach ($self->plugins) {
+        if ($_->isa('Open311::Endpoint::Integration::Multi')) {
+            push @plugins, $_ foreach $_->plugins;
+        } else {
+            push @plugins, $_;
+        }
+    }
+    my @urls = ();
+    foreach (@plugins) {
+        if ($_->can('get_integration')) {
+            my $config = $_->get_integration->config;
+            push @urls, $config->{api_url}          # Abavus, ATAK, Alloy, Boomi
+                || $config->{endpoint_url}          # Confirm, Ezytreev, Symology, Uniform, WDM
+                || $config->{endpoint}              # Salesforce
+                || $config->{url}                   # Echo, Whitespace
+                || $config->{jadu_api_base_url}     # Jadu
+                || $config->{collective_endpoint};  # Bartec
+        } elsif ($_->can('endpoint')) {
+            push @urls, $_->endpoint;
+        }
+    }
+    my %hosts;
+    foreach (grep { $_ } @urls) {
+        $hosts{URI->new($_)->host} = 1;
+    }
+    foreach (sort keys %hosts) {
+        my $check = `openssl s_client -connect $_:443 < /dev/null 2>/dev/null| openssl x509 -checkend 604800 -noout`;
+        next if $check =~ /will not expire/ && !$verbose;
+        print "$_: $check";
+    }
+}
+
 __PACKAGE__->run_if_script;
diff --git a/t/open311/endpoint/uk.t b/t/open311/endpoint/uk.t
index f78cb5780..3920934f4 100644
--- a/t/open311/endpoint/uk.t
+++ b/t/open311/endpoint/uk.t
@@ -18,6 +18,7 @@ test_multi(0, 'Open311::Endpoint::Integration::UK',
     'Open311::Endpoint::Integration::UK::Rutland' => 'rutland',
     'Open311::Endpoint::Integration::UK::Shropshire' => 'shropshire_confirm',
     'Open311::Endpoint::Integration::UK::Southwark' => 'southwark_confirm',
+    'Open311::Endpoint::Integration::UK::Surrey' => 'surrey_boomi',
     'Open311::Endpoint::Integration::UK::Sutton' => 'sutton_echo',
     'Open311::Endpoint::Integration::UK::Hampshire' => 'hampshire_confirm',
 );

From 41cff50f726452a771717638e096af361befef7f Mon Sep 17 00:00:00 2001
From: Matthew Somerville <matthew@mysociety.org>
Date: Tue, 12 Nov 2024 18:09:19 +0000
Subject: [PATCH 11/23] [Passthrough] Log calls made.

---
 perllib/Open311/Endpoint/Integration/Passthrough.pm | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/perllib/Open311/Endpoint/Integration/Passthrough.pm b/perllib/Open311/Endpoint/Integration/Passthrough.pm
index 4e7d1c440..6cde37968 100644
--- a/perllib/Open311/Endpoint/Integration/Passthrough.pm
+++ b/perllib/Open311/Endpoint/Integration/Passthrough.pm
@@ -18,6 +18,7 @@ use Moo;
 extends 'Open311::Endpoint';
 with 'Open311::Endpoint::Role::mySociety';
 with 'Open311::Endpoint::Role::ConfigFile';
+with 'Role::Logger';
 with 'Role::Memcached';
 
 use DateTime::Format::W3CDTF;
@@ -107,12 +108,16 @@ sub _request {
     my $resp;
     if ($method eq 'POST') {
         $params->{api_key} = $self->api_key;
+        $self->logger->debug($url);
+        $self->logger->dump($params);
         $resp = $self->ua->post($url, $params);
     } else {
         $url->query_form(%$params);
+        $self->logger->debug($url);
         $resp = $self->ua->get($url);
     }
     my $content = $resp->decoded_content;
+    $self->logger->debug($content);
     my $xml = $self->pt_xml->XMLin(\$content);
     die $xml->{error}[0]->{description} . "\n" if $xml->{error};
     return $xml;

From 247c2eaa5159240bdf644519754a75b256328e05 Mon Sep 17 00:00:00 2001
From: Matthew Somerville <matthew@mysociety.org>
Date: Fri, 8 Nov 2024 12:49:28 +0000
Subject: [PATCH 12/23] [Alloy] Stop reconstructing updated reports.

---
 .../Open311/Endpoint/Integration/AlloyV2.pm   | 179 +++++++-----------
 t/open311/endpoint/alloyv2.t                  |  56 ++----
 t/open311/endpoint/buckinghamshire_alloy.t    |  21 +-
 t/open311/endpoint/hackney_alloy.t            |  20 +-
 .../json/alloyv2/bucks_defects_search.json    |   2 +-
 ...4826965f30390f01cda_20230216134954750.json |  45 -----
 ...4826965f30390f01cda_20230216135008792.json |  54 ------
 .../endpoint/json/alloyv2/inspect_search.json |  20 +-
 .../customer_requests_query_response.json     |   2 +-
 .../reconstruct_123456_20231113110500.json    |  44 -----
 .../reconstruct_234567_20231113110500.json    |  44 -----
 .../reconstruct_234567_20231113111000.json    |  44 -----
 ...03824fc0de101a008fbb4f_20190101014844.json |  92 ---------
 .../reconstruct_3027029_20181231003240.json   |  33 ----
 .../reconstruct_3027029_20190101002240.json   |  33 ----
 .../reconstruct_3027029_20190101003240.json   |  33 ----
 .../reconstruct_3027030_20190101014240.json   |  33 ----
 .../reconstruct_3027031_20190101014340.json   |  33 ----
 .../reconstruct_3027032_20190101003240.json   |  33 ----
 .../reconstruct_3027034_20190101014913.json   |  33 ----
 .../reconstruct_3027052_20190106003240.json   |  33 ----
 .../reconstruct_4947501_20190101014813.json   |  92 ---------
 .../reconstruct_4947502_20190101015108.json   |  88 ---------
 .../reconstruct_5947505_20190104015108.json   |  96 ----------
 .../reconstruct_5947506_20190105013208.json   |  96 ----------
 .../hackney_environment/defects_search.json   |  14 +-
 ...ffffffae7015ac40c5b_20220420081926005.json |  91 ---------
 ...ffffffae7015ac40c5b_20220420104714612.json | 113 -----------
 t/open311/endpoint/northumberland_alloy.t     |  24 +--
 29 files changed, 123 insertions(+), 1378 deletions(-)
 delete mode 100644 t/open311/endpoint/json/alloyv2/bucks_reconstruct_63ee34826965f30390f01cda_20230216134954750.json
 delete mode 100644 t/open311/endpoint/json/alloyv2/bucks_reconstruct_63ee34826965f30390f01cda_20230216135008792.json
 delete mode 100644 t/open311/endpoint/json/alloyv2/northumberland/reconstruct_123456_20231113110500.json
 delete mode 100644 t/open311/endpoint/json/alloyv2/northumberland/reconstruct_234567_20231113110500.json
 delete mode 100644 t/open311/endpoint/json/alloyv2/northumberland/reconstruct_234567_20231113111000.json
 delete mode 100644 t/open311/endpoint/json/alloyv2/reconstruct_0203824fc0de101a008fbb4f_20190101014844.json
 delete mode 100644 t/open311/endpoint/json/alloyv2/reconstruct_3027029_20181231003240.json
 delete mode 100644 t/open311/endpoint/json/alloyv2/reconstruct_3027029_20190101002240.json
 delete mode 100644 t/open311/endpoint/json/alloyv2/reconstruct_3027029_20190101003240.json
 delete mode 100644 t/open311/endpoint/json/alloyv2/reconstruct_3027030_20190101014240.json
 delete mode 100644 t/open311/endpoint/json/alloyv2/reconstruct_3027031_20190101014340.json
 delete mode 100644 t/open311/endpoint/json/alloyv2/reconstruct_3027032_20190101003240.json
 delete mode 100644 t/open311/endpoint/json/alloyv2/reconstruct_3027034_20190101014913.json
 delete mode 100644 t/open311/endpoint/json/alloyv2/reconstruct_3027052_20190106003240.json
 delete mode 100644 t/open311/endpoint/json/alloyv2/reconstruct_4947501_20190101014813.json
 delete mode 100644 t/open311/endpoint/json/alloyv2/reconstruct_4947502_20190101015108.json
 delete mode 100644 t/open311/endpoint/json/alloyv2/reconstruct_5947505_20190104015108.json
 delete mode 100644 t/open311/endpoint/json/alloyv2/reconstruct_5947506_20190105013208.json
 delete mode 100644 t/open311/endpoint/json/hackney_environment/reconstruct_625ffffffffae7015ac40c5b_20220420081926005.json
 delete mode 100644 t/open311/endpoint/json/hackney_environment/reconstruct_625ffffffffae7015ac40c5b_20220420104714612.json

diff --git a/perllib/Open311/Endpoint/Integration/AlloyV2.pm b/perllib/Open311/Endpoint/Integration/AlloyV2.pm
index a495e8e51..f64919b7a 100644
--- a/perllib/Open311/Endpoint/Integration/AlloyV2.pm
+++ b/perllib/Open311/Endpoint/Integration/AlloyV2.pm
@@ -121,7 +121,7 @@ has service_whitelist => (
 
 =head2 update_store
 
-Directory for storing reconstructions of Alloy items to save on API calls
+Directory for storing some Alloy API calls to save on calls
 
 =cut
 
@@ -546,8 +546,8 @@ sub get_service_request_updates {
 =head3 'Inspections'
 
 'Inspection' updates fetch the C<rfs_design> resources that have been updated
-in the relevant time frame, fetching previous versions of each item.
-It uses C<inspection_attribute_mapping> for attribute mapping:
+in the relevant time frame. It uses C<inspection_attribute_mapping> for
+attribute mapping:
 
 =over 4
 
@@ -566,7 +566,7 @@ closure that can be used to override these at the end.
 
 =item C<inspector_comments>
 
-Used to find the attribute to look at to see if the comments have changed, for
+Used to find the attribute to look at to see if there are any comments, for
 providing as the text update.
 
 =back
@@ -587,72 +587,52 @@ sub _get_inspection_updates {
 
     my $assigned_to_users = $self->get_assigned_to_users(@$updates);
 
-    for my $update (@$updates) {
-        next unless $self->_accept_updated_resource($update);
+    for my $report (@$updates) {
+        next unless $self->_accept_updated_resource($report);
 
-        # We need to fetch all versions that changed in the time wanted
-        my @version_ids = $self->get_versions_of_resource($update->{itemId});
+        # The main result doesn't actually include the last edit date
+        my $date = $self->get_latest_version_date($report->{itemId});
+        my $update_dt = $self->date_to_truncated_dt($date);
+        next unless $update_dt >= $start_time && $update_dt <= $end_time;
 
-        my $last_description = '';
-        foreach my $date (@version_ids) {
-            my $description_to_send = '';
-            my $update_dt;
+        my $attributes = $self->alloy->attributes_to_hash($report);
 
-            # If we don't need to compare the comments field, we can
-            # consider updates only within the desired date range
-            if (!$mapping->{inspector_comments}) {
-                $update_dt = $self->date_to_truncated_dt( $date );
-                next unless $update_dt >= $start_time && $update_dt <= $end_time;
-            }
-
-            my $resource = $self->call_reconstruct($update->{itemId}, $date) or next;
-            my $attributes = $self->alloy->attributes_to_hash($resource);
-
-            my ($status, $reason_for_closure) = $self->_get_inspection_status($attributes, $mapping);
-
-            # If we are checking if the comments field has changed, we will
-            # have to fetch all the updates. Once we've fetched them we can
-            # throw away the ones that don't match the date range.
-            if ($mapping->{inspector_comments}) {
-                my $description = $attributes->{$mapping->{inspector_comments} || ''} || '';
-                # only want to put a description in the update if it's
-                # changed, so compare it to the last one.
-                $description_to_send = $description ne $last_description ? $description : '';
-                $last_description = $description;
-
-                $update_dt = $self->date_to_truncated_dt( $date );
-                next unless $update_dt >= $start_time && $update_dt <= $end_time;
-            }
+        my ($status, $reason_for_closure) = $self->_get_inspection_status($attributes, $mapping);
 
-            (my $id_date = $date) =~ s/\D//g;
-            my $id = $update->{itemId} . "_$id_date";
-
-            my %args = (
-                status => $status,
-                external_status_code => $reason_for_closure,
-                update_id => $resource->{signature},
-                service_request_id => $update->{itemId},
-                description => $description_to_send,
-                updated_datetime => $update_dt,
-            );
+        my $description = '';
+        if ($mapping->{inspector_comments}) {
+            $description = $attributes->{$mapping->{inspector_comments}} || '';
+        }
 
-            if ( my $assigned_to_user_id
-                = $attributes->{ $mapping->{assigned_to_user} // '' }[0] )
-            {
-                my $assigned_to_user
-                    = $assigned_to_users->{$assigned_to_user_id} ||= do {
-                    # There is a possibility the assigned-to user is not
-                    # already in the $assigned_to_users hash; do another
-                    # lookup if so
-                    $self->get_assigned_to_users($resource)
-                        ->{$assigned_to_user_id}
-                };
-
-                $args{extras} = $assigned_to_user if $assigned_to_user;
-            }
+        (my $id_date = $date) =~ s/\D//g;
+        my $id = $report->{itemId} . "_$id_date";
+
+        my %args = (
+            status => $status,
+            external_status_code => $reason_for_closure,
+            update_id => $id,
+            service_request_id => $report->{itemId},
+            description => $description,
+            updated_datetime => $update_dt,
+            extras => { latest_data_only => 1 },
+        );
 
-            push @updates, Open311::Endpoint::Service::Request::Update::mySociety->new( %args );
+        if ( my $assigned_to_user_id
+            = $attributes->{ $mapping->{assigned_to_user} // '' }[0] )
+        {
+            my $assigned_to_user
+                = $assigned_to_users->{$assigned_to_user_id} ||= do {
+                # There is a possibility the assigned-to user is not
+                # already in the $assigned_to_users hash; do another
+                # lookup if so
+                $self->get_assigned_to_users($report)
+                    ->{$assigned_to_user_id}
+            };
+
+            $args{extras} = { %{$args{extras}}, %$assigned_to_user } if $assigned_to_user;
         }
+
+        push @updates, Open311::Endpoint::Service::Request::Update::mySociety->new( %args );
     }
 
     return @updates;
@@ -722,11 +702,6 @@ Alloy value to a status.
 If the status is open or investigating and there is a linked defect, the update
 is skipped.
 
-Previous versions are fetched, as with 'inspections', their status worked out;
-there are no text attributes checked here. It has some special code that I'm
-not sure still applies, to prevent FMS adding phantom updates due to confusion
-over external status codes.
-
 =cut
 
 sub _get_defect_updates {
@@ -748,59 +723,37 @@ sub _get_defect_updates_resource {
     my $end_time = $self->date_to_dt($args->{end_date});
 
     my @updates;
-    my $closure_mapping = $self->config->{inspection_closure_mapping};
-    my %reverse_closure_mapping = map { $closure_mapping->{$_} => $_ } keys %{$closure_mapping};
-
     my $updates = $self->fetch_updated_resources($resource_name, $args->{start_date}, $args->{end_date});
-    for my $update (@$updates) {
-        next if $self->is_ignored_category( $update );
+    for my $report (@$updates) {
+        next if $self->is_ignored_category( $report );
 
         my $linked_defect;
-        my $attributes = $self->alloy->attributes_to_hash($update);
-
-        my $service_request_id = $update->{itemId};
+        my $attributes = $self->alloy->attributes_to_hash($report);
 
-        my $fms_id = $self->_get_defect_fms_id( $attributes );
+        my $service_request_id = $report->{itemId};
 
-        ($linked_defect, $service_request_id) = $self->_get_defect_inspection($update, $service_request_id);
-        $fms_id = undef if $linked_defect;
+        ($linked_defect, $service_request_id) = $self->_get_defect_inspection($report, $service_request_id);
 
         # we don't care about linked defects until they have been scheduled
         my $status = $self->defect_status($attributes);
         next if $self->_skip_job_update($linked_defect, $status);
 
-        my @version_ids = $self->get_versions_of_resource($update->{itemId});
-        foreach my $date (@version_ids) {
-            my $update_dt = $self->date_to_truncated_dt($date);
-            next unless $update_dt >= $start_time && $update_dt <= $end_time;
-
-            my $resource = $self->call_reconstruct($update->{itemId}, $date) or next;
-            my $attributes = $self->alloy->attributes_to_hash($resource);
-            my $status = $self->defect_status($attributes);
-
-            my %args = (
-                status => $status,
-                update_id => $resource->{signature},
-                service_request_id => $service_request_id,
-                description => '',
-                updated_datetime => $update_dt,
-            );
-
-            # we need to set this to stop phantom updates being produced. This happens because
-            # when an inspection is closed it always sets an external_status_code which we never
-            # unset. Then when updates arrive from defects with no external_status_code the template
-            # fetching code at FixMyStreet sees that the external_status_code has changed and fetches
-            # the template. This means we always get an update even if nothing has changed. So, set
-            # this to the external_status_code used when an inspection is marked for raising as a
-            # defect. Only do this for 'action_scheduled' thouogh as otherwise the template lookup
-            # will fail as it will be looking for status + ext code which won't match.
-            if ( $status eq 'action_scheduled' && ( $fms_id || $linked_defect ) ) {
-                $args{external_status_code} = $reverse_closure_mapping{'action_scheduled'};
-            }
-            $args{fixmystreet_id} = $fms_id if $fms_id;
+        my $date = $self->get_latest_version_date($report->{itemId});
+        my $update_dt = $self->date_to_truncated_dt($date);
+        next unless $update_dt >= $start_time && $update_dt <= $end_time;
+
+        (my $id_date = $date) =~ s/\D//g;
+        my $id = $report->{itemId} . "_$id_date";
+        my %args = (
+            status => $status,
+            update_id => $id,
+            service_request_id => $service_request_id,
+            description => '',
+            updated_datetime => $update_dt,
+            extras => { latest_data_only => 1 },
+        );
 
-            push @updates, Open311::Endpoint::Service::Request::Update::mySociety->new( %args );
-        }
+        push @updates, Open311::Endpoint::Service::Request::Update::mySociety->new( %args );
     }
 
     return @updates;
@@ -1000,7 +953,7 @@ sub service_request_id_for_resource {
     return $resource->{item}->{itemId};
 }
 
-sub get_versions_of_resource {
+sub get_latest_version_date {
     my ($self, $resource_id) = @_;
 
     my $versions = $self->alloy->api_call(call => "item-log/item/$resource_id")->{results};
@@ -1011,7 +964,7 @@ sub get_versions_of_resource {
     }
 
     @version_ids = sort(@version_ids);
-    return @version_ids;
+    return pop @version_ids;
 }
 
 sub date_to_dt {
@@ -1148,8 +1101,6 @@ sub process_attributes {
     return \@remapped;
 }
 
-sub _get_defect_fms_id { return undef; }
-
 sub _get_defect_inspection {
     my ($self, $defect, $service_request_id) = @_;
 
diff --git a/t/open311/endpoint/alloyv2.t b/t/open311/endpoint/alloyv2.t
index 057e0a49b..e3d7f192c 100644
--- a/t/open311/endpoint/alloyv2.t
+++ b/t/open311/endpoint/alloyv2.t
@@ -167,11 +167,6 @@ $integration->mock('api_call', sub {
             } else {
                 $content = path(__FILE__)->sibling('json/alloyv2/inspect_search.json')->slurp;
             }
-        } elsif ( $call =~ 'item-log/item/([^/]*)/reconstruct' ) {
-            my $id = $1;
-            my $date = $body->{date};
-            $date =~ s/\D//g;
-            $content = path(__FILE__)->sibling("json/alloyv2/reconstruct_${id}_$date.json")->slurp;
         }
     } else {
         if ( $call eq 'design/designs_enquiryInspectionRFS1001181_5d3245c5fe2ad806f8dfbaf6' ) {
@@ -498,8 +493,6 @@ subtest "create problem with no resource_id" => sub {
 };
 
 subtest "check fetch updates" => sub {
-  # Do it twice to check cache
-  for (1..2) {
     my $res = $endpoint->run_test_request(
       GET => '/servicerequestupdates.json?jurisdiction_id=dummy&start_date=2019-01-01T00:00:00Z&end_date=2019-03-01T02:00:00Z',
     );
@@ -510,64 +503,61 @@ subtest "check fetch updates" => sub {
 
     is_deeply decode_json($res->content),
     [ {
-        status => 'investigating',
-        service_request_id => '3027029',
-        description => '',
-        updated_datetime => '2019-01-01T00:22:40Z',
-        update_id => '5d32469bb4e1b90150014306',
-        media_url => '',
-    },
-    {
         status => 'investigating',
         service_request_id => '3027029',
         description => 'This is an updated customer response',
         updated_datetime => '2019-01-01T00:32:40Z',
-        update_id => '5d32469bb4e1b90150014305',
+        update_id => '3027029_20190101003240',
         media_url => '',
+        extras => { latest_data_only => 1 },
     },
     {
         status => 'investigating',
         service_request_id => '3027030',
         description => '',
         updated_datetime => '2019-01-01T01:42:40Z',
-        update_id => '5d32469bb4e1b90150014307',
+        update_id => '3027030_20190101014240',
         media_url => '',
+        extras => { latest_data_only => 1 },
     },
     {
         status => 'not_councils_responsibility',
         service_request_id => '3027031',
         description => '',
         updated_datetime => '2019-01-01T01:43:40Z',
-        update_id => '6d32469bb4e1b90150014305',
+        update_id => '3027031_20190101014340',
         media_url => '',
         external_status_code => '01b51bb5c0de101a004154b5',
+        extras => { latest_data_only => 1 },
     },
     {
         status => 'action_scheduled',
         service_request_id => '3027032',
         description => '',
         updated_datetime => '2019-01-01T01:48:13Z',
-        update_id => '5d324086b4e1b90150f946a0',
+        update_id => '4947501_20190101014813',
         media_url => '',
+        extras => { latest_data_only => 1 },
     },
     {
         status => 'investigating',
         service_request_id => '3027034',
         description => '',
         updated_datetime => '2019-01-01T01:49:13Z',
-        update_id => '5d324086b4e1b90150f946a1',
+        update_id => '3027034_20190101014913',
         media_url => '',
+        extras => { latest_data_only => 1 },
     },
     {
-        status => 'fixed',
+        status => 'open',
         service_request_id => '4947502',
         description => '',
         updated_datetime => '2019-01-01T01:51:08Z',
-        update_id => '5d324049b4e1b90150f94191',
+        update_id => '4947502_20190101015108',
         media_url => '',
+        extras => { latest_data_only => 1 },
     }
     ], 'correct json returned';
-  }
 };
 
 subtest "check fetch updates with cobrand skipping update where job has unchanged parent defect" => sub {
@@ -581,45 +571,41 @@ subtest "check fetch updates with cobrand skipping update where job has unchange
 
     is_deeply decode_json($res->content),
     [ {
-        status => 'investigating',
-        service_request_id => '3027029',
-        description => '',
-        updated_datetime => '2019-01-01T00:22:40Z',
-        update_id => '5d32469bb4e1b90150014306',
-        media_url => '',
-    },
-    {
         status => 'investigating',
         service_request_id => '3027029',
         description => 'This is an updated customer response',
         updated_datetime => '2019-01-01T00:32:40Z',
-        update_id => '5d32469bb4e1b90150014305',
+        update_id => '3027029_20190101003240',
         media_url => '',
+        extras => { latest_data_only => 1 },
     },
     {
         status => 'investigating',
         service_request_id => '3027030',
         description => '',
         updated_datetime => '2019-01-01T01:42:40Z',
-        update_id => '5d32469bb4e1b90150014307',
+        update_id => '3027030_20190101014240',
         media_url => '',
+        extras => { latest_data_only => 1 },
     },
     {
         status => 'not_councils_responsibility',
         service_request_id => '3027031',
         description => '',
         updated_datetime => '2019-01-01T01:43:40Z',
-        update_id => '6d32469bb4e1b90150014305',
+        update_id => '3027031_20190101014340',
         media_url => '',
         external_status_code => '01b51bb5c0de101a004154b5',
+        extras => { latest_data_only => 1 },
     },
     {
         status => 'investigating',
         service_request_id => '3027034',
         description => '',
         updated_datetime => '2019-01-01T01:49:13Z',
-        update_id => '5d324086b4e1b90150f946a1',
+        update_id => '3027034_20190101014913',
         media_url => '',
+        extras => { latest_data_only => 1 },
     }
     ], 'correct json returned';
 };
diff --git a/t/open311/endpoint/buckinghamshire_alloy.t b/t/open311/endpoint/buckinghamshire_alloy.t
index ea6d6e585..1a622e784 100644
--- a/t/open311/endpoint/buckinghamshire_alloy.t
+++ b/t/open311/endpoint/buckinghamshire_alloy.t
@@ -62,11 +62,6 @@ $integration->mock('api_call', sub {
             } elsif ($type eq 'designs_customerReportDefect_62e43ee75039cb015e3287e9') {
                 $content = path(__FILE__)->sibling('json/alloyv2/bucks_defects_search.json')->slurp;
             }
-        } elsif ( $call =~ 'item-log/item/([^/]*)/reconstruct' ) {
-            my $id = $1;
-            my $date = $body->{date};
-            $date =~ s/\D//g;
-            $content = path(__FILE__)->sibling("json/alloyv2/bucks_reconstruct_${id}_$date.json")->slurp;
         }
     } elsif ( $call eq 'design/designs_streetLights' ) {
         $content = path(__FILE__)->sibling('json/alloyv2/occ_design_resource.json')->slurp;
@@ -155,24 +150,16 @@ subtest "check fetch updates" => sub {
 
     is_deeply decode_json($res->content),
     [
-        {
-            description => "",
-            media_url => "",
-            service_request_id => "63ee34826965f30390f01cda",
-            status => "open",
-            external_status_code => '060',
-            update_id => "63ee34826965f30390f01ce3",
-            updated_datetime => "2023-02-16T13:49:54Z"
-        },
         {
             description => "",
             media_url => "",
             service_request_id => "63ee34826965f30390f01cda",
             status => "action_scheduled",
             external_status_code => '306',
-            update_id => "63ee3490b016c303ae032113",
-            updated_datetime => "2023-02-16T13:50:08Z"
-        }
+            update_id => "63ee34826965f30390f01cda_20230216135008792",
+            updated_datetime => "2023-02-16T13:50:08Z",
+            extras => { latest_data_only => 1 },
+        },
     ], 'correct json returned';
 };
 
diff --git a/t/open311/endpoint/hackney_alloy.t b/t/open311/endpoint/hackney_alloy.t
index c5021cad4..f2c3d2dbd 100644
--- a/t/open311/endpoint/hackney_alloy.t
+++ b/t/open311/endpoint/hackney_alloy.t
@@ -60,11 +60,6 @@ $integration->mock('api_call', sub {
             } elsif ($type eq 'designs_fixedMyStreetDefect') {
                 $content = path(__FILE__)->sibling('json/hackney_environment/defects_search.json')->slurp;
             }
-        } elsif ( $call =~ 'item-log/item/([^/]*)/reconstruct' ) {
-            my $id = $1;
-            my $date = $body->{date};
-            $date =~ s/\D//g;
-            $content = path(__FILE__)->sibling("json/hackney_environment/reconstruct_${id}_$date.json")->slurp;
         }
     } elsif ( $call =~ 'item-log/item/(.*)$' ) {
         $content = path(__FILE__)->sibling("json/hackney_environment/item_log_$1.json")->slurp;
@@ -158,23 +153,16 @@ subtest "check fetch updates" => sub {
 
     is_deeply decode_json($res->content),
     [
-        {
-            description => "",
-            media_url => "",
-            service_request_id => "625ffffffffae7015ac40c5b",
-            status => "open",
-            update_id => "625fffffff62a1016ce7f779",
-            updated_datetime => "2022-04-20T08:19:26Z"
-        },
         {
             description => "",
             media_url => "",
             service_request_id => "625ffffffffae7015ac40c5b",
             # this is derived from the Hackney Environment-specific defect_status code
             status => "not_councils_responsibility",
-            update_id => "625fe4b24edcdb01584733b2",
-            updated_datetime => "2022-04-20T10:47:14Z"
-        }
+            update_id => "625ffffffffae7015ac40c5b_20220420104714612",
+            updated_datetime => "2022-04-20T10:47:14Z",
+            extras => { latest_data_only => 1 },
+        },
     ], 'correct json returned';
 };
 
diff --git a/t/open311/endpoint/json/alloyv2/bucks_defects_search.json b/t/open311/endpoint/json/alloyv2/bucks_defects_search.json
index 182ab919f..6a4663dd6 100644
--- a/t/open311/endpoint/json/alloyv2/bucks_defects_search.json
+++ b/t/open311/endpoint/json/alloyv2/bucks_defects_search.json
@@ -31,7 +31,7 @@
         {
           "attributeCode": "attributes_customerReportDefectCustomerStatus_63690956d76320038c423af5",
           "value": [
-            "638610d11de74303a7ce77df"
+            "638610d11de74303a7ce77f3"
           ]
         }
       ],
diff --git a/t/open311/endpoint/json/alloyv2/bucks_reconstruct_63ee34826965f30390f01cda_20230216134954750.json b/t/open311/endpoint/json/alloyv2/bucks_reconstruct_63ee34826965f30390f01cda_20230216134954750.json
deleted file mode 100644
index bd60219bb..000000000
--- a/t/open311/endpoint/json/alloyv2/bucks_reconstruct_63ee34826965f30390f01cda_20230216134954750.json
+++ /dev/null
@@ -1,45 +0,0 @@
-{
-  "log": {
-    "itemId": "63ee34826965f30390f01cda",
-    "designCode": "designs_customerReportDefect_62e43ee75039cb015e3287e9",
-    "action": "Create",
-    "causes": [],
-    "date": "2023-02-16T13:49:54.750Z",
-    "username": "fixmystreetfixmystreet"
-  },
-  "item": {
-    "itemId": "63ee34826965f30390f01cda",
-    "designCode": "designs_customerReportDefect_62e43ee75039cb015e3287e9",
-    "attributes": [
-      {
-        "attributeCode": "attributes_customerReportDefectCRMReference_62e43eea0d2c1a0153b1c561",
-        "value": "123"
-      },
-      {
-        "attributeCode": "attributes_defectsDescription",
-        "value": "Test"
-      },
-      {
-        "attributeCode": "attributes_customerReportDefectReportedIssueText_636b830bd1026a0394a10de1",
-        "value": "Description"
-      },
-      {
-        "attributeCode": "attributes_defectsReportedDate",
-        "value": "2023-02-16T13:49:00.000Z"
-      },
-      {
-        "attributeCode": "attributes_customerReportDefectSubCategory_62e43eed0d2c1a0153b1c56e",
-        "value": [
-          "63d157ad3919a703aea03dae"
-        ]
-      },
-      {
-        "attributeCode": "attributes_customerReportDefectCustomerStatus_63690956d76320038c423af5",
-        "value": [
-          "638610d11de74303a7ce77df"
-        ]
-      }
-    ],
-    "signature": "63ee34826965f30390f01ce3"
-  }
-}
diff --git a/t/open311/endpoint/json/alloyv2/bucks_reconstruct_63ee34826965f30390f01cda_20230216135008792.json b/t/open311/endpoint/json/alloyv2/bucks_reconstruct_63ee34826965f30390f01cda_20230216135008792.json
deleted file mode 100644
index 8c29ed6d8..000000000
--- a/t/open311/endpoint/json/alloyv2/bucks_reconstruct_63ee34826965f30390f01cda_20230216135008792.json
+++ /dev/null
@@ -1,54 +0,0 @@
-{
-  "log": {
-    "itemId": "63ee34826965f30390f01cda",
-    "designCode": "designs_customerReportDefect_62e43ee75039cb015e3287e9",
-    "action": "Edit",
-    "causes": [
-      {
-        "workflowRunId": "63ee348be5b28203ae91632d",
-        "discriminator": "ItemChangeCauseWorkflowWebModel"
-      }
-    ],
-    "date": "2023-02-16T13:50:08.792Z",
-    "username": "alloybot"
-  },
-  "item": {
-    "itemId": "63ee34826965f30390f01cda",
-    "designCode": "designs_customerReportDefect_62e43ee75039cb015e3287e9",
-    "attributes": [
-      {
-        "attributeCode": "attributes_customerReportDefectCRMReference_62e43eea0d2c1a0153b1c561",
-        "value": "123"
-      },
-      {
-        "attributeCode": "attributes_defectsDescription",
-        "value": "Test"
-      },
-      {
-        "attributeCode": "attributes_customerReportDefectReportedIssueText_636b830bd1026a0394a10de1",
-        "value": "Description"
-      },
-      {
-        "attributeCode": "attributes_defectsReportedDate",
-        "value": "2023-02-16T13:49:00.000Z"
-      },
-      {
-        "attributeCode": "attributes_customerReportDefectSubCategory_62e43eed0d2c1a0153b1c56e",
-        "value": [
-          "63d157ad3919a703aea03dae"
-        ]
-      },
-      {
-        "attributeCode": "attributes_customerReportDefectCustomerStatus_63690956d76320038c423af5",
-        "value": [
-          "638610d11de74303a7ce77f3"
-        ]
-      }
-    ],
-    "signature": "63ee3490b016c303ae032113"
-  },
-  "previous": {
-    "attributes": [],
-    "signature": "63ee34826965f30390f01ce3"
-  }
-}
diff --git a/t/open311/endpoint/json/alloyv2/inspect_search.json b/t/open311/endpoint/json/alloyv2/inspect_search.json
index 06d427926..2b72e86b3 100644
--- a/t/open311/endpoint/json/alloyv2/inspect_search.json
+++ b/t/open311/endpoint/json/alloyv2/inspect_search.json
@@ -17,12 +17,12 @@
       "colour": "#99cf22",
       "attributes":[
         {
-          "attributeCode": "attributes_tastStatus",
-          "value": [""]
+          "attributeCode": "attributes_taskStatus",
+          "value": ["5bc5bdd281d088d177342c73"]
         },
         {
           "attributeCode": "attributes_enquiryInspectionRFS1001181ResponseToCustomer1011096_5d3245dafe2ad806f8dfbb2e",
-          "value": ""
+          "value": "This is an updated customer response"
         },
         {
           "attributeCode": "attributes_enquiryInspectionRFS1001181ReasonForClosure1010926_5d3245d8fe2ad806f8dfbb24",
@@ -47,8 +47,8 @@
       "colour": "#99cf22",
       "attributes":[
         {
-          "attributeCode": "attributes_tastStatus",
-          "value": [""]
+          "attributeCode": "attributes_taskStatus",
+          "value": ["5bc5bdd281d088d177342c74"]
         },
         {
           "attributeCode": "attributes_enquiryInspectionRFS1001181ResponseToCustomer1011096_5d3245dafe2ad806f8dfbb2e",
@@ -77,8 +77,8 @@
       "colour": "#99cf22",
       "attributes":[
         {
-          "attributeCode": "attributes_tastStatus",
-          "value": [""]
+          "attributeCode": "attributes_taskStatus",
+          "value": ["5bc5bdd281d088d177342c76"]
         },
         {
           "attributeCode": "attributes_enquiryInspectionRFS1001181ResponseToCustomer1011096_5d3245dafe2ad806f8dfbb2e",
@@ -86,7 +86,7 @@
         },
         {
           "attributeCode": "attributes_enquiryInspectionRFS1001181ReasonForClosure1010926_5d3245d8fe2ad806f8dfbb24",
-          "value": [""]
+          "value": ["01b51bb5c0de101a004154b5"]
         }
       ],
       "signature": "5d32469bb4e1b90150014305",
@@ -107,8 +107,8 @@
       "colour": "#99cf22",
       "attributes":[
         {
-          "attributeCode": "attributes_tastStatus",
-          "value": [""]
+          "attributeCode": "attributes_taskStatus",
+          "value": ["5bc5bdd281d088d177342c73"]
         },
         {
           "attributeCode": "attributes_enquiryInspectionRFS1001181ResponseToCustomer1011096_5d3245dafe2ad806f8dfbb2e",
diff --git a/t/open311/endpoint/json/alloyv2/northumberland/customer_requests_query_response.json b/t/open311/endpoint/json/alloyv2/northumberland/customer_requests_query_response.json
index b3a7d53e8..0eb745374 100644
--- a/t/open311/endpoint/json/alloyv2/northumberland/customer_requests_query_response.json
+++ b/t/open311/endpoint/json/alloyv2/northumberland/customer_requests_query_response.json
@@ -77,7 +77,7 @@
         {
           "attributeCode": "attributes_customerRequestAssignedTo_653664b0557119eef53a97e1",
           "value": [
-            "234"
+            "345"
           ]
         }
       ]
diff --git a/t/open311/endpoint/json/alloyv2/northumberland/reconstruct_123456_20231113110500.json b/t/open311/endpoint/json/alloyv2/northumberland/reconstruct_123456_20231113110500.json
deleted file mode 100644
index 833493d30..000000000
--- a/t/open311/endpoint/json/alloyv2/northumberland/reconstruct_123456_20231113110500.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
-  "item": {
-    "collection": "Live",
-    "designCode": "designs_customerRequest_6386279ffb3d97038c4e03a9",
-    "itemId": "123456",
-    "signature": "100001",
-    "attributes": [
-      {
-        "attributeCode": "attributes_customerRequestFMSSummary_63862bd505cb250393c204d7",
-        "value": "Something is broken"
-      },
-      {
-        "attributeCode": "attributes_customerRequestFMSDescription_63862bed05cb250393c2096d",
-        "value": "Please fix it"
-      },
-      {
-        "attributeCode": "attributes_customerRequestFixMyStreetID_63862c38bafbd20397883f72",
-        "value": "5000063"
-      },
-      {
-        "attributeCode": "attributes_customerRequestRequestCategory_63862851fb3d97038c4e1cfc",
-        "value": [
-          "61fb016c4c5c56015448093f"
-        ]
-      },
-      {
-        "attributeCode": "attributes_customerRequestFMSStatus_6391f99705cb2503934eb858",
-        "value": [
-          "6391f9639e005f039665d65a"
-        ]
-      },
-      {
-        "attributeCode": "attributes_customerRequestEmail_63868f209e005f0396e519ca",
-        "value": "person@email.com"
-      },
-      {
-        "attributeCode": "attributes_customerRequestAssignedTo_653664b0557119eef53a97e1",
-        "value": [
-          "123"
-        ]
-      }
-    ]
-  }
-}
diff --git a/t/open311/endpoint/json/alloyv2/northumberland/reconstruct_234567_20231113110500.json b/t/open311/endpoint/json/alloyv2/northumberland/reconstruct_234567_20231113110500.json
deleted file mode 100644
index f7d8f3d7c..000000000
--- a/t/open311/endpoint/json/alloyv2/northumberland/reconstruct_234567_20231113110500.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
-  "item": {
-    "collection": "Live",
-    "designCode": "designs_customerRequest_6386279ffb3d97038c4e03a9",
-    "itemId": "234567",
-    "signature": "200002",
-    "attributes": [
-      {
-        "attributeCode": "attributes_customerRequestFMSSummary_63862bd505cb250393c204d7",
-        "value": "Something is bothering me"
-      },
-      {
-        "attributeCode": "attributes_customerRequestFMSDescription_63862bed05cb250393c2096d",
-        "value": "Please help me"
-      },
-      {
-        "attributeCode": "attributes_customerRequestFixMyStreetID_63862c38bafbd20397883f72",
-        "value": "5000063"
-      },
-      {
-        "attributeCode": "attributes_customerRequestRequestCategory_63862851fb3d97038c4e1cfc",
-        "value": [
-          "61fb016c4c5c56015448093f"
-        ]
-      },
-      {
-        "attributeCode": "attributes_customerRequestFMSStatus_6391f99705cb2503934eb858",
-        "value": [
-          "6391f9639e005f039665d65a"
-        ]
-      },
-      {
-        "attributeCode": "attributes_customerRequestEmail_63868f209e005f0396e519ca",
-        "value": "person@email.com"
-      },
-      {
-        "attributeCode": "attributes_customerRequestAssignedTo_653664b0557119eef53a97e1",
-        "value": [
-          "234"
-        ]
-      }
-    ]
-  }
-}
diff --git a/t/open311/endpoint/json/alloyv2/northumberland/reconstruct_234567_20231113111000.json b/t/open311/endpoint/json/alloyv2/northumberland/reconstruct_234567_20231113111000.json
deleted file mode 100644
index e83364a63..000000000
--- a/t/open311/endpoint/json/alloyv2/northumberland/reconstruct_234567_20231113111000.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
-  "item": {
-    "collection": "Live",
-    "designCode": "designs_customerRequest_6386279ffb3d97038c4e03a9",
-    "itemId": "234567",
-    "signature": "200002",
-    "attributes": [
-      {
-        "attributeCode": "attributes_customerRequestFMSSummary_63862bd505cb250393c204d7",
-        "value": "Something is bothering me"
-      },
-      {
-        "attributeCode": "attributes_customerRequestFMSDescription_63862bed05cb250393c2096d",
-        "value": "Please help me"
-      },
-      {
-        "attributeCode": "attributes_customerRequestFixMyStreetID_63862c38bafbd20397883f72",
-        "value": "5000063"
-      },
-      {
-        "attributeCode": "attributes_customerRequestRequestCategory_63862851fb3d97038c4e1cfc",
-        "value": [
-          "61fb016c4c5c56015448093f"
-        ]
-      },
-      {
-        "attributeCode": "attributes_customerRequestFMSStatus_6391f99705cb2503934eb858",
-        "value": [
-          "6391f9639e005f039665d65a"
-        ]
-      },
-      {
-        "attributeCode": "attributes_customerRequestEmail_63868f209e005f0396e519ca",
-        "value": "person@email.com"
-      },
-      {
-        "attributeCode": "attributes_customerRequestAssignedTo_653664b0557119eef53a97e1",
-        "value": [
-          "345"
-        ]
-      }
-    ]
-  }
-}
diff --git a/t/open311/endpoint/json/alloyv2/reconstruct_0203824fc0de101a008fbb4f_20190101014844.json b/t/open311/endpoint/json/alloyv2/reconstruct_0203824fc0de101a008fbb4f_20190101014844.json
deleted file mode 100644
index 69ed05d49..000000000
--- a/t/open311/endpoint/json/alloyv2/reconstruct_0203824fc0de101a008fbb4f_20190101014844.json
+++ /dev/null
@@ -1,92 +0,0 @@
-{
-  "item":
-    {
-      "itemId": "4947501",
-      "designCode": "designs_ancillaryItems1000793_5d324047fe2ad806f8dfb464",
-      "collection": "Live",
-      "start": "2019-01-01T00:00:00.723Z",
-      "end": "9999-12-31T23:59:59.999Z",
-      "geometry": {
-        "type": "Point",
-        "coordinates": [
-          -0.5878147518991453,
-          52.31519007307925
-        ]
-      },
-      "icon": "icon-defect-full",
-      "colour": "#AAC6Df",
-      "attributes": [
-        {
-          "attributeCode": "attributes_defectsReportedDate",
-          "value": "2019-06-20T14:34:00.000Z"
-        },
-        {
-          "attributeCode": "attributes_defectsStatus",
-          "value": [
-            "5c8bdfc88ae862230019dc22"
-          ]
-        },
-        {
-          "attributeCode": "attributes_defectsDefectNumber",
-          "value": 203262
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793OriginalDefectNumber1003788_5d324051fe2ad806f8dfb46c",
-          "value": "DEF-00203262"
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793TeamMembers1005688_5d324052fe2ad806f8dfb471",
-          "value": [
-            "0203a7d0c0de101a008fe0d0"
-          ]
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793DefectType1005689_5d324052fe2ad806f8dfb476",
-          "value": [
-            "018fae47c0de101a001be747"
-          ]
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793TreatmentType1005690_5d324053fe2ad806f8dfb47b",
-          "value": [
-            "018facffc0de101a001be5ff"
-          ]
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793RepairType1005691_5d324054fe2ad806f8dfb480",
-          "value": [
-            "01854946c0de101a00118246"
-          ]
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793Priorities1005692_5d324055fe2ad806f8dfb485",
-          "value": [
-            "01b51bbac0de101a004154ba"
-          ]
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793LocationDescription1006369_5d324057fe2ad806f8dfb48f",
-          "value": "A6 Bypass Higham Ferrers - from Chowns Mill roundabout to Kimbolton Road"
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793PermRepairDescription1006371_5d324059fe2ad806f8dfb499",
-          "value": "Repair VRS barrier - wire rope posts"
-        },
-        {
-          "attributeCode": "attributes_defectsRemediedDate",
-          "value": "2019-06-26T13:23:00.000Z"
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793ActivityCode1011097_5d32405afe2ad806f8dfb49e",
-          "value": "AI"
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793TargetDate1011490_5d32405bfe2ad806f8dfb4a3",
-          "value": "2019-12-19T00:00:00.000Z"
-        }
-      ],
-      "signature": "5d324086b4e1b90150f946a0",
-      "title": "DEF-3401",
-      "subtitle": "Ancillary Items"
-    }
-}
diff --git a/t/open311/endpoint/json/alloyv2/reconstruct_3027029_20181231003240.json b/t/open311/endpoint/json/alloyv2/reconstruct_3027029_20181231003240.json
deleted file mode 100644
index c4f3f14a0..000000000
--- a/t/open311/endpoint/json/alloyv2/reconstruct_3027029_20181231003240.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
-  "item":
-    {
-      "itemId": "3027029",
-      "designCode": "designs_enquiryInspectionRFS1001181_5d3245c5fe2ad806f8dfbaf6",
-      "collection": "Live",
-      "start": "2019-01-01T00:32:40Z",
-      "end": "9999-12-31T23:59:59.999Z",
-      "geometry":{
-        "type": "Point",
-        "coordinates":[-0.89868522337640622, 52.112591554183062]
-      },
-      "icon": "icon-inspection-unknown",
-      "colour": "#99cf22",
-      "attributes":[
-        {
-          "attributeCode": "attributes_taskStatus",
-          "value": ["5bc5bdd281d088d177342c73"]
-        },
-        {
-          "attributeCode": "attributes_enquiryInspectionRFS1001181ResponseToCustomer1011096_5d3245dafe2ad806f8dfbb2e",
-          "value": "This is a customer response"
-        },
-        {
-          "attributeCode": "attributes_enquiryInspectionRFS1001181ReasonForClosure1010926_5d3245d8fe2ad806f8dfbb24",
-          "value": [""]
-        }
-      ],
-      "signature": "5d32469bb4e1b90150014305",
-      "title": "INS-34032, 1584451",
-      "subtitle": "Enquiry Inspection RFS"
-    }
-}
diff --git a/t/open311/endpoint/json/alloyv2/reconstruct_3027029_20190101002240.json b/t/open311/endpoint/json/alloyv2/reconstruct_3027029_20190101002240.json
deleted file mode 100644
index 1b44a5a2d..000000000
--- a/t/open311/endpoint/json/alloyv2/reconstruct_3027029_20190101002240.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
-  "item":
-    {
-      "itemId": "3027029",
-      "designCode": "designs_enquiryInspectionRFS1001181_5d3245c5fe2ad806f8dfbaf6",
-      "collection": "Live",
-      "start": "2019-01-01T00:32:40Z",
-      "end": "9999-12-31T23:59:59.999Z",
-      "geometry":{
-        "type": "Point",
-        "coordinates":[-0.89868522337640622, 52.112591554183062]
-      },
-      "icon": "icon-inspection-unknown",
-      "colour": "#99cf22",
-      "attributes":[
-        {
-          "attributeCode": "attributes_taskStatus",
-          "value": ["5bc5bdd281d088d177342c73"]
-        },
-        {
-          "attributeCode": "attributes_enquiryInspectionRFS1001181ResponseToCustomer1011096_5d3245dafe2ad806f8dfbb2e",
-          "value": "This is a customer response"
-        },
-        {
-          "attributeCode": "attributes_enquiryInspectionRFS1001181ReasonForClosure1010926_5d3245d8fe2ad806f8dfbb24",
-          "value": [""]
-        }
-      ],
-      "signature": "5d32469bb4e1b90150014306",
-      "title": "INS-34032, 1584451",
-      "subtitle": "Enquiry Inspection RFS"
-    }
-}
diff --git a/t/open311/endpoint/json/alloyv2/reconstruct_3027029_20190101003240.json b/t/open311/endpoint/json/alloyv2/reconstruct_3027029_20190101003240.json
deleted file mode 100644
index b238bcc81..000000000
--- a/t/open311/endpoint/json/alloyv2/reconstruct_3027029_20190101003240.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
-  "item":
-    {
-      "itemId": "3027029",
-      "designCode": "designs_enquiryInspectionRFS1001181_5d3245c5fe2ad806f8dfbaf6",
-      "collection": "Live",
-      "start": "2019-01-01T00:32:40Z",
-      "end": "9999-12-31T23:59:59.999Z",
-      "geometry":{
-        "type": "Point",
-        "coordinates":[-0.89868522337640622, 52.112591554183062]
-      },
-      "icon": "icon-inspection-unknown",
-      "colour": "#99cf22",
-      "attributes":[
-        {
-          "attributeCode": "attributes_taskStatus",
-          "value": ["5bc5bdd281d088d177342c73"]
-        },
-        {
-          "attributeCode": "attributes_enquiryInspectionRFS1001181ResponseToCustomer1011096_5d3245dafe2ad806f8dfbb2e",
-          "value": "This is an updated customer response"
-        },
-        {
-          "attributeCode": "attributes_enquiryInspectionRFS1001181ReasonForClosure1010926_5d3245d8fe2ad806f8dfbb24",
-          "value": [""]
-        }
-      ],
-      "signature": "5d32469bb4e1b90150014305",
-      "title": "INS-34032, 1584451",
-      "subtitle": "Enquiry Inspection RFS"
-    }
-}
diff --git a/t/open311/endpoint/json/alloyv2/reconstruct_3027030_20190101014240.json b/t/open311/endpoint/json/alloyv2/reconstruct_3027030_20190101014240.json
deleted file mode 100644
index f256e2315..000000000
--- a/t/open311/endpoint/json/alloyv2/reconstruct_3027030_20190101014240.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
-  "item":
-    {
-      "itemId": "3027029",
-      "designCode": "designs_enquiryInspectionRFS1001181_5d3245c5fe2ad806f8dfbaf6",
-      "collection": "Live",
-      "start": "2019-01-01T01:42:40Z",
-      "end": "9999-12-31T23:59:59.999Z",
-      "geometry":{
-        "type": "Point",
-        "coordinates":[-0.89868522337640622, 52.112591554183062]
-      },
-      "icon": "icon-inspection-unknown",
-      "colour": "#99cf22",
-      "attributes":[
-        {
-          "attributeCode": "attributes_taskStatus",
-          "value": ["5bc5bdd281d088d177342c74"]
-        },
-        {
-          "attributeCode": "attributes_enquiryInspectionRFS1001181ResponseToCustomer1011096_5d3245dafe2ad806f8dfbb2e",
-          "value": ""
-        },
-        {
-          "attributeCode": "attributes_enquiryInspectionRFS1001181ReasonForClosure1010926_5d3245d8fe2ad806f8dfbb24",
-          "value": [""]
-        }
-      ],
-      "signature": "5d32469bb4e1b90150014307",
-      "title": "INS-34032, 1584451",
-      "subtitle": "Enquiry Inspection RFS"
-    }
-}
diff --git a/t/open311/endpoint/json/alloyv2/reconstruct_3027031_20190101014340.json b/t/open311/endpoint/json/alloyv2/reconstruct_3027031_20190101014340.json
deleted file mode 100644
index 9993d7186..000000000
--- a/t/open311/endpoint/json/alloyv2/reconstruct_3027031_20190101014340.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
-  "item":
-    {
-      "itemId": "3027029",
-      "designCode": "designs_enquiryInspectionRFS1001181_5d3245c5fe2ad806f8dfbaf6",
-      "collection": "Live",
-      "start": "2019-01-01T01:43:40Z",
-      "end": "9999-12-31T23:59:59.999Z",
-      "geometry":{
-        "type": "Point",
-        "coordinates":[-0.89868522337640622, 52.112591554183062]
-      },
-      "icon": "icon-inspection-unknown",
-      "colour": "#99cf22",
-      "attributes":[
-        {
-          "attributeCode": "attributes_taskStatus",
-          "value": ["5bc5bdd281d088d177342c76"]
-        },
-        {
-          "attributeCode": "attributes_enquiryInspectionRFS1001181ResponseToCustomer1011096_5d3245dafe2ad806f8dfbb2e",
-          "value": ""
-        },
-        {
-          "attributeCode": "attributes_enquiryInspectionRFS1001181ReasonForClosure1010926_5d3245d8fe2ad806f8dfbb24",
-          "value": ["01b51bb5c0de101a004154b5"]
-        }
-      ],
-      "signature": "6d32469bb4e1b90150014305",
-      "title": "INS-34032, 1584451",
-      "subtitle": "Enquiry Inspection RFS"
-    }
-}
diff --git a/t/open311/endpoint/json/alloyv2/reconstruct_3027032_20190101003240.json b/t/open311/endpoint/json/alloyv2/reconstruct_3027032_20190101003240.json
deleted file mode 100644
index 6c37a30cf..000000000
--- a/t/open311/endpoint/json/alloyv2/reconstruct_3027032_20190101003240.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
-  "item":
-    {
-      "itemId": "3027032",
-      "designCode": "designs_enquiryInspectionRFS1001181_5d3245c5fe2ad806f8dfbaf6",
-      "collection": "Live",
-      "start": "2019-01-01T00:32:40Z",
-      "end": "9999-12-31T23:59:59.999Z",
-      "geometry":{
-        "type": "Point",
-        "coordinates":[-0.89868522337640622, 52.112591554183062]
-      },
-      "icon": "icon-inspection-unknown",
-      "colour": "#99cf22",
-      "attributes":[
-        {
-          "attributeCode": "attributes_taskStatus",
-          "value": ["01b51bb6c0de101a004154b7"]
-        },
-        {
-          "attributeCode": "attributes_enquiryInspectionRFS1001181ResponseToCustomer1011096_5d3245dafe2ad806f8dfbb2e",
-          "value": "This is a customer response"
-        },
-        {
-          "attributeCode": "attributes_enquiryInspectionRFS1001181ReasonForClosure1010926_5d3245d8fe2ad806f8dfbb24",
-          "value": [""]
-        }
-      ],
-      "signature": "5d32469bb4e1b90150014307",
-      "title": "INS-34032, 1584451",
-      "subtitle": "Enquiry Inspection RFS"
-    }
-}
diff --git a/t/open311/endpoint/json/alloyv2/reconstruct_3027034_20190101014913.json b/t/open311/endpoint/json/alloyv2/reconstruct_3027034_20190101014913.json
deleted file mode 100644
index 2d35bf462..000000000
--- a/t/open311/endpoint/json/alloyv2/reconstruct_3027034_20190101014913.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
-  "item":
-    {
-      "itemId": "3027034",
-      "designCode": "designs_enquiryInspectionRFS1001181_5d3245c5fe2ad806f8dfbaf6",
-      "collection": "Live",
-      "start": "2019-01-01T00:32:40Z",
-      "end": "9999-12-31T23:59:59.999Z",
-      "geometry":{
-        "type": "Point",
-        "coordinates":[-0.89868522337640622, 52.112591554183062]
-      },
-      "icon": "icon-inspection-unknown",
-      "colour": "#99cf22",
-      "attributes":[
-        {
-          "attributeCode": "attributes_taskStatus",
-          "value": ["5bc5bdd281d088d177342c73"]
-        },
-        {
-          "attributeCode": "attributes_enquiryInspectionRFS1001181ResponseToCustomer1011096_5d3245dafe2ad806f8dfbb2e",
-          "value": ""
-        },
-        {
-          "attributeCode": "attributes_enquiryInspectionRFS1001181ReasonForClosure1010926_5d3245d8fe2ad806f8dfbb24",
-          "value": [""]
-        }
-      ],
-      "signature": "5d324086b4e1b90150f946a1",
-      "title": "INS-34032, 1584451",
-      "subtitle": "Enquiry Inspection RFS"
-    }
-}
diff --git a/t/open311/endpoint/json/alloyv2/reconstruct_3027052_20190106003240.json b/t/open311/endpoint/json/alloyv2/reconstruct_3027052_20190106003240.json
deleted file mode 100644
index 482978d17..000000000
--- a/t/open311/endpoint/json/alloyv2/reconstruct_3027052_20190106003240.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
-  "item":
-    {
-      "itemId": "3027052",
-      "designCode": "designs_enquiryInspectionRFS1001181_5d3245c5fe2ad806f8dfbaf6",
-      "collection": "Live",
-      "start": "2019-01-06T00:32:40Z",
-      "end": "9999-12-31T23:59:59.999Z",
-      "geometry":{
-        "type": "Point",
-        "coordinates":[-0.89868522337640622, 52.112591554183062]
-      },
-      "icon": "icon-inspection-unknown",
-      "colour": "#99cf22",
-      "attributes":[
-        {
-          "attributeCode": "attributes_taskStatus",
-          "value": ["5bc5bdd281d088d177342c76"]
-        },
-        {
-          "attributeCode": "attributes_enquiryInspectionRFS1001181ResponseToCustomer1011096_5d3245dafe2ad806f8dfbb2e",
-          "value": "This is a customer response"
-        },
-        {
-          "attributeCode": "attributes_enquiryInspectionRFS1001181ReasonForClosure1010926_5d3245d8fe2ad806f8dfbb24",
-          "value": ["01b51bb6c0de101a004154b7"]
-        }
-      ],
-      "signature": "5d32469bb4e1b90150014308",
-      "title": "INS-34032, 1584451",
-      "subtitle": "Enquiry Inspection RFS"
-    }
-}
diff --git a/t/open311/endpoint/json/alloyv2/reconstruct_4947501_20190101014813.json b/t/open311/endpoint/json/alloyv2/reconstruct_4947501_20190101014813.json
deleted file mode 100644
index 8823ca806..000000000
--- a/t/open311/endpoint/json/alloyv2/reconstruct_4947501_20190101014813.json
+++ /dev/null
@@ -1,92 +0,0 @@
-{
-  "item":
-    {
-      "itemId": "4947501",
-      "designCode": "designs_ancillaryItems1000793_5d324047fe2ad806f8dfb464",
-      "collection": "Live",
-      "start": "2019-01-01T00:00:00.723Z",
-      "end": "9999-12-31T23:59:59.999Z",
-      "geometry": {
-        "type": "Point",
-        "coordinates": [
-          -0.5878147518991453,
-          52.31519007307925
-        ]
-      },
-      "icon": "icon-defect-full",
-      "colour": "#AAC6Df",
-      "attributes": [
-        {
-          "attributeCode": "attributes_defectsReportedDate",
-          "value": "2019-06-20T14:34:00.000Z"
-        },
-        {
-          "attributeCode": "attributes_defectsStatus",
-          "value": [
-            "5c8bdfc18ae862230019dc21"
-          ]
-        },
-        {
-          "attributeCode": "attributes_defectsDefectNumber",
-          "value": 203262
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793OriginalDefectNumber1003788_5d324051fe2ad806f8dfb46c",
-          "value": "DEF-00203262"
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793TeamMembers1005688_5d324052fe2ad806f8dfb471",
-          "value": [
-            "0203a7d0c0de101a008fe0d0"
-          ]
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793DefectType1005689_5d324052fe2ad806f8dfb476",
-          "value": [
-            "018fae47c0de101a001be747"
-          ]
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793TreatmentType1005690_5d324053fe2ad806f8dfb47b",
-          "value": [
-            "018facffc0de101a001be5ff"
-          ]
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793RepairType1005691_5d324054fe2ad806f8dfb480",
-          "value": [
-            "01854946c0de101a00118246"
-          ]
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793Priorities1005692_5d324055fe2ad806f8dfb485",
-          "value": [
-            "01b51bbac0de101a004154ba"
-          ]
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793LocationDescription1006369_5d324057fe2ad806f8dfb48f",
-          "value": "A6 Bypass Higham Ferrers - from Chowns Mill roundabout to Kimbolton Road"
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793PermRepairDescription1006371_5d324059fe2ad806f8dfb499",
-          "value": "Repair VRS barrier - wire rope posts"
-        },
-        {
-          "attributeCode": "attributes_defectsRemediedDate",
-          "value": "2019-06-26T13:23:00.000Z"
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793ActivityCode1011097_5d32405afe2ad806f8dfb49e",
-          "value": "AI"
-        },
-        {
-          "attributeCode": "attributes_ancillaryItems1000793TargetDate1011490_5d32405bfe2ad806f8dfb4a3",
-          "value": "2019-12-19T00:00:00.000Z"
-        }
-      ],
-      "signature": "5d324086b4e1b90150f946a0",
-      "title": "DEF-3401",
-      "subtitle": "Ancillary Items"
-    }
-}
diff --git a/t/open311/endpoint/json/alloyv2/reconstruct_4947502_20190101015108.json b/t/open311/endpoint/json/alloyv2/reconstruct_4947502_20190101015108.json
deleted file mode 100644
index 796f843e6..000000000
--- a/t/open311/endpoint/json/alloyv2/reconstruct_4947502_20190101015108.json
+++ /dev/null
@@ -1,88 +0,0 @@
-{
-  "item":
-    {
-      "itemId": "4947502",
-      "designCode": "designs_benches1000794_5d32400ffe2ad80354bbdae6",
-      "collection": "Live",
-      "start": "2019-01-01T00:00:00.002Z",
-      "end": "9999-12-31T23:59:59.999Z",
-      "geometry": {
-        "type": "Point",
-        "coordinates": [
-          -0.9050079189236898,
-          52.236715671161356
-        ]
-      },
-      "icon": "icon-defect-full",
-      "colour": "#AAC6Df",
-      "attributes": [
-        {
-          "attributeCode": "attributes_defectsDescription",
-          "value": "Broken Northampton Borough Council benches x 2"
-        },
-        {
-          "attributeCode": "attributes_defectsReportedDate",
-          "value": "2019-04-26T06:35:00.000Z"
-        },
-        {
-          "attributeCode": "attributes_defectsStatus",
-          "value": [
-            "5c8bdfc88ae862230019dc22"
-          ]
-        },
-        {
-          "attributeCode": "attributes_defectsDefectNumber",
-          "value": 200842
-        },
-        {
-          "attributeCode": "attributes_benches1000794OriginalDefectNumber1003788_5d324018fe2ad80354bbdaee",
-          "value": "DEF-00200842"
-        },
-        {
-          "attributeCode": "attributes_benches1000794TeamMembers1005709_5d324018fe2ad80354bbdaf3",
-          "value": [
-            "02037ebbc0de101a008fb7bb"
-          ]
-        },
-        {
-          "attributeCode": "attributes_benches1000794DefectType1005710_5d32401afe2ad80354bbdaf8",
-          "value": [
-            "018fae6ac0de101a001be76a"
-          ]
-        },
-        {
-          "attributeCode": "attributes_benches1000794RepairType1005712_5d32401bfe2ad80354bbdafd",
-          "value": [
-            "01854946c0de101a00118246"
-          ]
-        },
-        {
-          "attributeCode": "attributes_benches1000794Priorities1005713_5d32401cfe2ad80354bbdb02",
-          "value": [
-            "01b51bb8c0de101a004154b8"
-          ]
-        },
-        {
-          "attributeCode": "attributes_benches1000794LocationDescription1006378_5d32401efe2ad80354bbdb0c",
-          "value": "On centre island, Black Lion Hill"
-        },
-        {
-          "attributeCode": "attributes_benches1000794PermRepairDescription1006380_5d324020fe2ad80354bbdb16",
-          "value": "Emailed Hub to refer to Northampton Borough Council for renewal/repair"
-        },
-        {
-          "attributeCode": "attributes_benches1000794TreatmentType1007311_5d324021fe2ad80354bbdb1b",
-          "value": [
-            "018fad07c0de101a001be607"
-          ]
-        },
-        {
-          "attributeCode": "attributes_benches1000794ActivityCode1011100_5d324022fe2ad80354bbdb20",
-          "value": "BE"
-        }
-      ],
-      "signature": "5d324049b4e1b90150f94191",
-      "title": "DEF-3400",
-      "subtitle": "Benches"
-    }
-}
diff --git a/t/open311/endpoint/json/alloyv2/reconstruct_5947505_20190104015108.json b/t/open311/endpoint/json/alloyv2/reconstruct_5947505_20190104015108.json
deleted file mode 100644
index c6061b1e4..000000000
--- a/t/open311/endpoint/json/alloyv2/reconstruct_5947505_20190104015108.json
+++ /dev/null
@@ -1,96 +0,0 @@
-{
-  "item":
-    {
-      "itemId": "5947505",
-      "designCode": "designs_shelterDamaged1000793_5d324047fe2ad806f8dfb464",
-      "collection": "Live",
-      "start": "2019-01-01T00:00:00.723Z",
-      "end": "9999-12-31T23:59:59.999Z",
-      "geometry": {
-        "type": "Point",
-        "coordinates": [
-          -0.5878147518991453,
-          52.31519007307925
-        ]
-      },
-      "icon": "icon-defect-full",
-      "colour": "#AAC6Df",
-      "attributes": [
-        {
-          "attributeCode": "attributes_defectsReportedDate",
-          "value": "2019-06-20T14:34:00.000Z"
-        },
-        {
-          "attributeCode": "attributes_defectsStatus",
-          "value": [
-            "5c8bdfc18ae862230019dc21"
-          ]
-        },
-        {
-          "attributeCode": "attributes_defectsDefectNumber",
-          "value": 203262
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793OriginalDefectNumber1003788_5d324051fe2ad806f8dfb46c",
-          "value": "DEF-00203262"
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793TeamMembers1005688_5d324052fe2ad806f8dfb471",
-          "value": [
-            "0203a7d0c0de101a008fe0d0"
-          ]
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793DefectType1005689_5d324052fe2ad806f8dfb476",
-          "value": [
-            "018fae47c0de101a001be747"
-          ]
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793TreatmentType1005690_5d324053fe2ad806f8dfb47b",
-          "value": [
-            "018facffc0de101a001be5ff"
-          ]
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793RepairType1005691_5d324054fe2ad806f8dfb480",
-          "value": [
-            "01854946c0de101a00118246"
-          ]
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793Priorities1005692_5d324055fe2ad806f8dfb485",
-          "value": [
-            "01b51bbac0de101a004154ba"
-          ]
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793LocationDescription1006369_5d324057fe2ad806f8dfb48f",
-          "value": "A6 Bypass Higham Ferrers - from Chowns Mill roundabout to Kimbolton Road"
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793PermRepairDescription1006371_5d324059fe2ad806f8dfb499",
-          "value": "Repair VRS barrier - wire rope posts"
-        },
-        {
-          "attributeCode": "attributes_defectsRemediedDate",
-          "value": "2019-06-26T13:23:00.000Z"
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793ActivityCode1011097_5d32405afe2ad806f8dfb49e",
-          "value": "AI"
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793TargetDate1011490_5d32405bfe2ad806f8dfb4a3",
-          "value": "2019-12-19T00:00:00.000Z"
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1001018StreetDoctorID1011599_5d324270fe2ad806f8dfb738",
-          "value": "10101010"
-        }
-      ],
-      "signature": "5d324086b4e1b90150f946a1",
-      "title": "DEF-3401",
-      "subtitle": "Ancillary Items"
-    }
-}
diff --git a/t/open311/endpoint/json/alloyv2/reconstruct_5947506_20190105013208.json b/t/open311/endpoint/json/alloyv2/reconstruct_5947506_20190105013208.json
deleted file mode 100644
index 08ff4eff8..000000000
--- a/t/open311/endpoint/json/alloyv2/reconstruct_5947506_20190105013208.json
+++ /dev/null
@@ -1,96 +0,0 @@
-{
-  "item":
-    {
-      "itemId": "5947506",
-      "designCode": "designs_shelterDamaged1000793_5d324047fe2ad806f8dfb464",
-      "collection": "Live",
-      "start": "2019-01-01T00:00:00.723Z",
-      "end": "9999-12-31T23:59:59.999Z",
-      "geometry": {
-        "type": "Point",
-        "coordinates": [
-          -0.5878147518991453,
-          52.31519007307925
-        ]
-      },
-      "icon": "icon-defect-full",
-      "colour": "#AAC6Df",
-      "attributes": [
-        {
-          "attributeCode": "attributes_defectsReportedDate",
-          "value": "2019-06-20T14:34:00.000Z"
-        },
-        {
-          "attributeCode": "attributes_defectsStatus",
-          "value": [
-            "5c8bdfc18ae862230019dc21"
-          ]
-        },
-        {
-          "attributeCode": "attributes_defectsDefectNumber",
-          "value": 203262
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793OriginalDefectNumber1003788_5d324051fe2ad806f8dfb46c",
-          "value": "DEF-00203262"
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793TeamMembers1005688_5d324052fe2ad806f8dfb471",
-          "value": [
-            "0203a7d0c0de101a008fe0d0"
-          ]
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793DefectType1005689_5d324052fe2ad806f8dfb476",
-          "value": [
-            "018fae47c0de101a001be747"
-          ]
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793TreatmentType1005690_5d324053fe2ad806f8dfb47b",
-          "value": [
-            "018facffc0de101a001be5ff"
-          ]
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793RepairType1005691_5d324054fe2ad806f8dfb480",
-          "value": [
-            "01854946c0de101a00118246"
-          ]
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793Priorities1005692_5d324055fe2ad806f8dfb485",
-          "value": [
-            "01b51bbac0de101a004154ba"
-          ]
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793LocationDescription1006369_5d324057fe2ad806f8dfb48f",
-          "value": "A6 Bypass Higham Ferrers - from Chowns Mill roundabout to Kimbolton Road"
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793PermRepairDescription1006371_5d324059fe2ad806f8dfb499",
-          "value": "Repair VRS barrier - wire rope posts"
-        },
-        {
-          "attributeCode": "attributes_defectsRemediedDate",
-          "value": "2019-06-26T13:23:00.000Z"
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793ActivityCode1011097_5d32405afe2ad806f8dfb49e",
-          "value": "AI"
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1000793TargetDate1011490_5d32405bfe2ad806f8dfb4a3",
-          "value": "2019-12-19T00:00:00.000Z"
-        },
-        {
-          "attributeCode": "attributes_shelterDamaged1001018StreetDoctorID1011599_5d324270fe2ad806f8dfb738",
-          "value": "10101010"
-        }
-      ],
-      "signature": "5d324086b4e1b90150f946b2",
-      "title": "DEF-3401",
-      "subtitle": "Ancillary Items"
-    }
-}
diff --git a/t/open311/endpoint/json/hackney_environment/defects_search.json b/t/open311/endpoint/json/hackney_environment/defects_search.json
index f5f86cb46..6cd1e0c64 100644
--- a/t/open311/endpoint/json/hackney_environment/defects_search.json
+++ b/t/open311/endpoint/json/hackney_environment/defects_search.json
@@ -53,12 +53,24 @@
                         "abcdea5823d4d9016616c151"
                     ]
                 },
+                {
+                    "attributeCode": "attributes_fixedMyStreetDefectFixMyStreetCancellationReason",
+                    "value": [
+                        "617807ffffffff015d18e75f"
+                    ]
+                },
                 {
                     "attributeCode": "attributes_defectsStatus",
                     "value": [
                         "5c8bdfc88ae862230019dc22"
                     ]
                 },
+                 {
+                "attributeCode": "attributes_tasksStatus",
+                "value": [
+                    "5bc5bffffffff8d177342c79"
+                ]
+            },
                 {
                     "attributeCode": "attributes_defectsDefectNumber",
                     "value": 7379
@@ -83,4 +95,4 @@
             "signature": "625fe4b24edcdb01584733b2"
         }
    ]
-}
\ No newline at end of file
+}
diff --git a/t/open311/endpoint/json/hackney_environment/reconstruct_625ffffffffae7015ac40c5b_20220420081926005.json b/t/open311/endpoint/json/hackney_environment/reconstruct_625ffffffffae7015ac40c5b_20220420081926005.json
deleted file mode 100644
index f46ebf0bc..000000000
--- a/t/open311/endpoint/json/hackney_environment/reconstruct_625ffffffffae7015ac40c5b_20220420081926005.json
+++ /dev/null
@@ -1,91 +0,0 @@
-{
-    "log": {
-        "itemId": "625ffffffffae7015ac40c5b",
-        "designCode": "designs_fixedMyStreetDefect",
-        "action": "Edit",
-        "causes": [],
-        "date": "2022-04-20T08:19:26.005Z",
-        "username": "fmsuser"
-    },
-    "item": {
-        "itemId": "625ffffffffae7015ac40c5b",
-        "designCode": "designs_fixedMyStreetDefect",
-        "collection": "Live",
-        "locked": false,
-        "attributes": [
-            {
-                "attributeCode": "attributes_defectsDescription",
-                "value": "This is just a test"
-            },
-            {
-                "attributeCode": "attributes_fixedMyStreetDefectFMSReportTitle",
-                "value": "Test title"
-            },
-            {
-                "attributeCode": "attributes_defectsReportedDate",
-                "value": "2022-04-19T19:17:00.000Z"
-            },
-            {
-                "attributeCode": "attributes_fixedMyStreetDefectFMSID",
-                "value": "14482"
-            },
-            {
-                "attributeCode": "attributes_fixedMyStreetDefectFMSNearestAddress",
-                "value": "2 Hillman Street, Hackney, E8 1EA"
-            },
-            {
-                "attributeCode": "attributes_itemsGeometry",
-                "value": {
-                    "type": "Point",
-                    "coordinates": [
-                        -0.056875,
-                        51.544915
-                    ]
-                }
-            },
-            {
-                "attributeCode": "attributes_fixedMyStreetDefectFMSCategory",
-                "value": [
-                    "613f20e5227cd101577025e2"
-                ]
-            },
-            {
-                "attributeCode": "attributes_fixedMyStreetDefectContactDetails",
-                "value": [
-                    "6172aa5823d4d9016616c151"
-                ]
-            },
-            {
-                "attributeCode": "attributes_defectsDefectNumber",
-                "value": 7379
-            },
-            {
-                "attributeCode": "attributes_itemsTitle",
-                "value": " 7379 - 14482 - 2 Hillman Street, Hackney, E8 1EA"
-            },
-            {
-                "attributeCode": "attributes_itemsSubtitle",
-                "value": "Fly Tipping"
-            },
-            {
-                "attributeCode": "attributes_fixedMyStreetDefectFixMyStreetNotes",
-                "value": "Customer update at 2022-04-19 20:17:17\nHi,\r\n\r\nThank you for your report related to Littering & Cleanliness. We aim to resolve reports like this by the end of the next working day (exc weekends and bank holidays).\r\n\r\nThank you\r\nHackney Council\r\n"
-            },
-            {
-                "attributeCode": "attributes_defectsStatus",
-                "value": [
-                    "5c8bdfb28ae862230019dc1e"
-                ]
-            }
-        ],
-        "signature": "625fffffff62a1016ce7f779"
-    },
-    "previous": {
-        "attributes": [
-            {
-                "attributeCode": "attributes_fixedMyStreetDefectFixMyStreetNotes"
-            }
-        ],
-        "signature": "625f0ac061fae7015ac40c60"
-    }
-}
\ No newline at end of file
diff --git a/t/open311/endpoint/json/hackney_environment/reconstruct_625ffffffffae7015ac40c5b_20220420104714612.json b/t/open311/endpoint/json/hackney_environment/reconstruct_625ffffffffae7015ac40c5b_20220420104714612.json
deleted file mode 100644
index e6048e2d8..000000000
--- a/t/open311/endpoint/json/hackney_environment/reconstruct_625ffffffffae7015ac40c5b_20220420104714612.json
+++ /dev/null
@@ -1,113 +0,0 @@
-{
-    "log": {
-        "itemId": "625ffffffffae7015ac40c5b",
-        "designCode": "designs_fixedMyStreetDefect",
-        "action": "Edit",
-        "causes": [],
-        "date": "2022-04-20T10:47:14.612Z",
-        "username": "staffuser"
-    },
-    "item": {
-        "itemId": "625ffffffffae7015ac40c5b",
-        "designCode": "designs_fixedMyStreetDefect",
-        "collection": "Live",
-        "locked": false,
-        "attributes": [
-            {
-                "attributeCode": "attributes_defectsDescription",
-                "value": "This is just a test"
-            },
-            {
-                "attributeCode": "attributes_fixedMyStreetDefectFMSReportTitle",
-                "value": "Test title"
-            },
-            {
-                "attributeCode": "attributes_defectsReportedDate",
-                "value": "2022-04-19T19:17:00.000Z"
-            },
-            {
-                "attributeCode": "attributes_fixedMyStreetDefectFMSID",
-                "value": "14482"
-            },
-            {
-                "attributeCode": "attributes_fixedMyStreetDefectFMSNearestAddress",
-                "value": "2 Hillman Street, Hackney, E8 1EA"
-            },
-            {
-                "attributeCode": "attributes_itemsGeometry",
-                "value": {
-                    "type": "Point",
-                    "coordinates": [
-                        -0.056875,
-                        51.544915
-                    ]
-                }
-            },
-            {
-                "attributeCode": "attributes_fixedMyStreetDefectFMSCategory",
-                "value": [
-                    "613f20e5227cd101577025e2"
-                ]
-            },
-            {
-                "attributeCode": "attributes_fixedMyStreetDefectContactDetails",
-                "value": [
-                    "6172aa5823d4d9016616c151"
-                ]
-            },
-            {
-                "attributeCode": "attributes_fixedMyStreetDefectFixMyStreetCancellationReason",
-                "value": [
-                    "617807ffffffff015d18e75f"
-                ]
-            },
-            {
-                "attributeCode": "attributes_defectsStatus",
-                "value": [
-                    "5c8bdfc88ae862230019dc22"
-                ]
-            },
-            {
-                "attributeCode": "attributes_tasksStatus",
-                "value": [
-                    "5bc5bffffffff8d177342c79"
-                ]
-            },
-            {
-                "attributeCode": "attributes_defectsDefectNumber",
-                "value": 7379
-            },
-            {
-                "attributeCode": "attributes_itemsTitle",
-                "value": " 7379 - 14482 - 2 Hillman Street, Hackney, E8 1EA"
-            },
-            {
-                "attributeCode": "attributes_itemsSubtitle",
-                "value": "Fly Tipping"
-            },
-            {
-                "attributeCode": "attributes_fixedMyStreetDefectFixMyStreetNotes",
-                "value": "Customer update at 2022-04-19 20:17:17\nHi,\r\n\r\nThank you for your report related to Littering & Cleanliness. We aim to resolve reports like this by the end of the next working day (exc weekends and bank holidays).\r\n\r\nThank you\r\nHackney Council\r\n"
-            },
-            {
-                "attributeCode": "attributes_defectsRemediedDate",
-                "value": "2022-04-20T09:30:00.000Z"
-            }
-        ],
-        "signature": "625fe4b24edcdb01584733b2"
-    },
-    "previous": {
-        "attributes": [
-            {
-                "attributeCode": "attributes_defectsStatus",
-                "value": [
-                    "5c8bdfb28ae862230019dc1e"
-                ]
-            },
-            {
-                "attributeCode": "attributes_defectsRemediedDate"
-            }
-        ],
-        "signature": "625fffffff62a1016ce7f779"
-    }
-}
\ No newline at end of file
diff --git a/t/open311/endpoint/northumberland_alloy.t b/t/open311/endpoint/northumberland_alloy.t
index 4853ff131..0aa75ff4d 100644
--- a/t/open311/endpoint/northumberland_alloy.t
+++ b/t/open311/endpoint/northumberland_alloy.t
@@ -72,13 +72,6 @@ $integration->mock('api_call', sub {
         # Looking up asset.
         $content = path(__FILE__)->sibling("json/alloyv2/northumberland_asset_lookup_response.json")->slurp;
 
-    } elsif ($body && $call =~ 'item-log/item/([^/]*)/reconstruct') {
-        my $id   = $1;
-        my $date = $body->{date};
-        $date =~ s/\D//g;
-        $content
-            = path(__FILE__)
-            ->sibling("json/alloyv2/northumberland/reconstruct_${id}_$date.json")->slurp;
     } elsif ($call =~ 'item-log/item/([^/]*)') {
         my $id = $1;
         $content
@@ -400,9 +393,10 @@ subtest "check fetch updates" => sub {
             media_url          => '',
             service_request_id => '123456',
             status             => 'open',
-            update_id          => '100001',
+            update_id          => '123456_20231113110500',
             updated_datetime   => '2023-11-13T11:05:00Z',
             extras             => {
+                latest_data_only => 1,
                 assigned_user_name  => 'FMS User 123',
                 assigned_user_email => '123@email.com',
             },
@@ -411,20 +405,10 @@ subtest "check fetch updates" => sub {
             media_url          => '',
             service_request_id => '234567',
             status             => 'open',
-            update_id          => '200002',
-            updated_datetime   => '2023-11-13T11:05:00Z',
-            extras             => {
-                assigned_user_name  => 'FMS User 234',
-                assigned_user_email => '234@email.com',
-            },
-        },
-        {   description        => '',
-            media_url          => '',
-            service_request_id => '234567',
-            status             => 'open',
-            update_id          => '200002',
+            update_id          => '234567_20231113111000',
             updated_datetime   => '2023-11-13T11:10:00Z',
             extras             => {
+                latest_data_only => 1,
                 assigned_user_name  => 'FMS User 345',
                 assigned_user_email => '345@email.com',
             },

From e65a5c4a72ff7d7eb5f971c83344e718bbc28985 Mon Sep 17 00:00:00 2001
From: Matthew Somerville <matthew@mysociety.org>
Date: Fri, 8 Nov 2024 12:50:33 +0000
Subject: [PATCH 13/23] [Bucks] Cache status API call lookup.

---
 .../Integration/UK/Buckinghamshire/Alloy.pm         | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/perllib/Open311/Endpoint/Integration/UK/Buckinghamshire/Alloy.pm b/perllib/Open311/Endpoint/Integration/UK/Buckinghamshire/Alloy.pm
index cd80b0dae..ec8e4d805 100644
--- a/perllib/Open311/Endpoint/Integration/UK/Buckinghamshire/Alloy.pm
+++ b/perllib/Open311/Endpoint/Integration/UK/Buckinghamshire/Alloy.pm
@@ -10,6 +10,7 @@ package Open311::Endpoint::Integration::UK::Buckinghamshire::Alloy;
 
 use Moo;
 extends 'Open311::Endpoint::Integration::AlloyV2';
+with 'Role::Memcached';
 
 around BUILDARGS => sub {
     my ($orig, $class, %args) = @_;
@@ -97,10 +98,14 @@ sub _get_inspection_status {
         my $status_code = $attributes->{$mapping->{status}}->[0];
         $status = $self->inspection_status($status_code);
 
-        my $status_obj = $self->alloy->api_call(call => "item/$status_code");
-        $status_obj = $status_obj->{item};
-        my $status_attributes = $self->alloy->attributes_to_hash($status_obj);
-        $ext_code = $status_attributes->{$mapping->{external_status_code}};
+        $ext_code = $self->memcache->get("alloy-item-$status_code");
+        unless ($ext_code) {
+            my $status_obj = $self->alloy->api_call(call => "item/$status_code");
+            $status_obj = $status_obj->{item};
+            my $status_attributes = $self->alloy->attributes_to_hash($status_obj);
+            $ext_code = $status_attributes->{$mapping->{external_status_code}};
+            $self->memcache->set("alloy-item-$status_code", $ext_code, 86400);
+        }
     }
     return ($status, $ext_code);
 }

From a953b5d1e82bfd81493f8cc48cd307fe5e0f245c Mon Sep 17 00:00:00 2001
From: Matthew Somerville <matthew@mysociety.org>
Date: Fri, 8 Nov 2024 12:34:51 +0000
Subject: [PATCH 14/23] [Alloy] Increase per-page to 100.

---
 perllib/Integrations/AlloyV2.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perllib/Integrations/AlloyV2.pm b/perllib/Integrations/AlloyV2.pm
index 05fb9fcec..c2afe11bd 100644
--- a/perllib/Integrations/AlloyV2.pm
+++ b/perllib/Integrations/AlloyV2.pm
@@ -184,7 +184,7 @@ sub search {
     return [] unless $result_count;
 
     my $maxPages = 100;
-    my $pageSize = $result_count <= 2000 ? 20 : ( ceil($result_count / $maxPages) + 1 );
+    my $pageSize = $result_count <= 10000 ? 100 : ( ceil($result_count / $maxPages) + 1 );
     my $pages = int( $result_count / $pageSize ) + 1;
 
     my $query_body = $body_base;

From d3e28535d7e9e8682812ea96cae25e1326302671 Mon Sep 17 00:00:00 2001
From: Victoria Mihell-Hale <victoria.mihellhale@hotmail.co.uk>
Date: Thu, 24 Oct 2024 15:50:28 +0100
Subject: [PATCH 15/23] [Bexley][WW] Implement worksheet sending for delivery
 requests

---
 perllib/Integrations/Whitespace.pm            |  8 +++--
 .../Endpoint/Integration/Whitespace.pm        |  5 +++
 .../Endpoint/Service/UKCouncil/Whitespace.pm  |  8 ++++-
 t/open311/endpoint/bexley.t                   | 34 +++++++++++++++++++
 t/open311/endpoint/whitespace.t               |  1 +
 5 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/perllib/Integrations/Whitespace.pm b/perllib/Integrations/Whitespace.pm
index d432336d5..ab68edeba 100644
--- a/perllib/Integrations/Whitespace.pm
+++ b/perllib/Integrations/Whitespace.pm
@@ -100,9 +100,13 @@ sub CreateWorksheet {
         die "No service mapping found for $params->{service_item_name}";
     }
 
+    my $service_id = $params->{service_code} eq 'request_new_container'
+        ? $service_params->{delivery_service_id}
+        : $service_params->{service_id};
+
     my $worksheet = ixhash(
         Uprn => $params->{uprn},
-        ServiceId => $service_params->{service_id},
+        ServiceId => $service_id,
         WorksheetReference => $params->{worksheet_reference},
         WorksheetMessage => $params->{worksheet_message},
         ServiceItemInputs => ixhash(
@@ -110,7 +114,7 @@ sub CreateWorksheet {
                 ixhash(
                     'wsap:ServiceItemId' => $service_params->{service_item_id},
                     'wsap:ServiceItemName' => '',
-                    'wsap:ServiceItemQuantity' => 1,
+                    'wsap:ServiceItemQuantity' => $params->{quantity},
                 )
             ]
         ),
diff --git a/perllib/Open311/Endpoint/Integration/Whitespace.pm b/perllib/Open311/Endpoint/Integration/Whitespace.pm
index f923b0787..54db69acb 100644
--- a/perllib/Open311/Endpoint/Integration/Whitespace.pm
+++ b/perllib/Open311/Endpoint/Integration/Whitespace.pm
@@ -78,13 +78,18 @@ sub post_service_request {
 
     my $integration = $self->get_integration;
 
+    $args->{attributes}{location_of_containers} //= '';
+    $args->{attributes}{quantity} ||= 1;
+
     my $worksheet_id = $integration->CreateWorksheet({
+        service_code => $args->{service_code},
         uprn => $args->{attributes}->{uprn},
         service_item_name => $args->{attributes}->{service_item_name},
         worksheet_reference => $args->{attributes}->{fixmystreet_id},
         worksheet_message => $self->_worksheet_message($args),
         assisted_yn => $args->{attributes}->{assisted_yn},
         location_of_containers => $args->{attributes}->{location_of_containers},
+        quantity => $args->{attributes}->{quantity},
     });
 
     my $request = $self->new_request(
diff --git a/perllib/Open311/Endpoint/Service/UKCouncil/Whitespace.pm b/perllib/Open311/Endpoint/Service/UKCouncil/Whitespace.pm
index ca9dc06d4..3ebf1e5fc 100644
--- a/perllib/Open311/Endpoint/Service/UKCouncil/Whitespace.pm
+++ b/perllib/Open311/Endpoint/Service/UKCouncil/Whitespace.pm
@@ -46,10 +46,16 @@ sub _build_attributes {
             required => 0,
             automated => 'hidden_field',
         ),
+        Open311::Endpoint::Service::Attribute->new(
+            code => 'quantity',
+            description => 'Number of containers',
+            datatype => 'string',
+            required => 0,
+            automated => 'hidden_field',
+        ),
     );
 
     return \@attributes;
 }
 
 1;
-
diff --git a/t/open311/endpoint/bexley.t b/t/open311/endpoint/bexley.t
index 1f96537d1..02d8718e4 100644
--- a/t/open311/endpoint/bexley.t
+++ b/t/open311/endpoint/bexley.t
@@ -228,6 +228,7 @@ subtest "Whitespace worksheets use Bexley's custom message" => sub {
     my $ws = Test::MockModule->new('Integrations::Whitespace');
     $ws->mock(CreateWorksheet => sub {
         my ($self, $args) = @_;
+        is $args->{quantity}, 1;
         is $args->{worksheet_message}, "Assisted collection? Yes\n\nLocation of containers: location of containers\n";
         return 1001;
     });
@@ -255,4 +256,37 @@ subtest "Whitespace worksheets use Bexley's custom message" => sub {
         } ], 'correct json returned';
 };
 
+subtest "Whitespace worksheet, no location, with quantity" => sub {
+    my $ws = Test::MockModule->new('Integrations::Whitespace');
+    $ws->mock(CreateWorksheet => sub {
+        my ($self, $args) = @_;
+        is $args->{quantity}, 3;
+        is $args->{location_of_containers}, '';
+        is $args->{worksheet_message}, "Assisted collection? No\n\nLocation of containers: \n";
+        return 1002;
+    });
+    my $res = $endpoint->run_test_request(
+        POST => '/requests.json',
+        api_key => 'test',
+        service_code => 'Whitespace-WS2',
+        first_name => 'Bob',
+        last_name => 'Mould',
+        description => "This is the details",
+        lat => 51,
+        long => -1,
+        'attribute[uprn]' => 1000001,
+        'attribute[service_item_name]' => 'service item name',
+        'attribute[fixmystreet_id]' => 'fixmystreet id',
+        'attribute[assisted_yn]' => 'No',
+        'attribute[quantity]' => 3,
+    );
+    ok $res->is_success, 'valid request'
+        or diag $res->content;
+
+    is_deeply decode_json($res->content),
+        [ {
+            "service_request_id" => "Whitespace-1002"
+        } ], 'correct json returned';
+};
+
 done_testing;
diff --git a/t/open311/endpoint/whitespace.t b/t/open311/endpoint/whitespace.t
index 164d46d16..4ddb29372 100644
--- a/t/open311/endpoint/whitespace.t
+++ b/t/open311/endpoint/whitespace.t
@@ -109,6 +109,7 @@ subtest "GET service" => sub {
           { code => 'fixmystreet_id', order => 3, required => 'true', variable => 'false', datatype => 'string', datatype_description => '', automated => 'server_set', description => 'external system ID' },
           { code => 'assisted_yn', order => 4, required => 'false', variable => 'true', datatype => 'string', datatype_description => '', automated => 'hidden_field', description => 'Assisted collection (Yes/No)' },
           { code => 'location_of_containers', order => 5, required => 'false', variable => 'true', datatype => 'string', datatype_description => '', automated => 'hidden_field', description => 'Location of containers' },
+          { code => 'quantity', order => 6, required => 'false', variable => 'true', datatype => 'string', datatype_description => '', automated => 'hidden_field', description => 'Number of containers' },
       ],
     }, 'correct json returned';
 };

From 765ff9321e1c7231392561018fb5571561d9a1b3 Mon Sep 17 00:00:00 2001
From: Victoria Mihell-Hale <victoria.mihellhale@hotmail.co.uk>
Date: Tue, 29 Oct 2024 11:01:25 +0000
Subject: [PATCH 16/23] [Bexley][WW] Use collection_service_id for removal
 requests

---
 perllib/Integrations/Whitespace.pm | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/perllib/Integrations/Whitespace.pm b/perllib/Integrations/Whitespace.pm
index ab68edeba..817521313 100644
--- a/perllib/Integrations/Whitespace.pm
+++ b/perllib/Integrations/Whitespace.pm
@@ -102,6 +102,8 @@ sub CreateWorksheet {
 
     my $service_id = $params->{service_code} eq 'request_new_container'
         ? $service_params->{delivery_service_id}
+        : $params->{service_code} eq 'request_container_removal'
+        ? $service_params->{collection_service_id}
         : $service_params->{service_id};
 
     my $worksheet = ixhash(

From c7a023616ea6b064d2cb4a66dbba933ef600353f Mon Sep 17 00:00:00 2001
From: Victoria Mihell-Hale <victoria.mihellhale@hotmail.co.uk>
Date: Wed, 13 Nov 2024 10:48:07 +0000
Subject: [PATCH 17/23] [Bexley][WW] Include location_of_letterbox in
 Whitespace worksheets

---
 perllib/Integrations/Whitespace.pm                     |  6 ++++++
 .../Endpoint/Integration/UK/Bexley/Whitespace.pm       | 10 ++++++++--
 perllib/Open311/Endpoint/Integration/Whitespace.pm     |  2 ++
 .../Open311/Endpoint/Service/UKCouncil/Whitespace.pm   |  7 +++++++
 t/open311/endpoint/whitespace.t                        |  3 ++-
 5 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/perllib/Integrations/Whitespace.pm b/perllib/Integrations/Whitespace.pm
index 817521313..03f2d1c43 100644
--- a/perllib/Integrations/Whitespace.pm
+++ b/perllib/Integrations/Whitespace.pm
@@ -133,6 +133,12 @@ sub CreateWorksheet {
                     'wsap:ServicePropertyValue' => $params->{location_of_containers},
                 ),
             },
+            $params->{location_of_letterbox} ? ({
+                'wsap:Input.CreateWorksheetInput.ServicePropertyInput' => ixhash(
+                    'wsap:ServicePropertyId' => 82,
+                    'wsap:ServicePropertyValue' => $params->{location_of_letterbox},
+                ),
+            }) : (),
         ],
     );
 
diff --git a/perllib/Open311/Endpoint/Integration/UK/Bexley/Whitespace.pm b/perllib/Open311/Endpoint/Integration/UK/Bexley/Whitespace.pm
index 20a7b785d..ad587de5f 100644
--- a/perllib/Open311/Endpoint/Integration/UK/Bexley/Whitespace.pm
+++ b/perllib/Open311/Endpoint/Integration/UK/Bexley/Whitespace.pm
@@ -12,8 +12,14 @@ around BUILDARGS => sub {
 sub _worksheet_message {
     my ($self, $args) = @_;
 
-    return "Assisted collection? $args->{attributes}->{assisted_yn}\n\n" .
-           "Location of containers: $args->{attributes}->{location_of_containers}\n";
+    my $msg = "Assisted collection? $args->{attributes}->{assisted_yn}\n\n"
+        . "Location of containers: $args->{attributes}->{location_of_containers}\n";
+
+    $msg
+        .= "\nLocation of letterbox: $args->{attributes}->{location_of_letterbox}\n"
+        if $args->{attributes}->{location_of_letterbox};
+
+    return $msg;
 }
 
 __PACKAGE__->run_if_script;
diff --git a/perllib/Open311/Endpoint/Integration/Whitespace.pm b/perllib/Open311/Endpoint/Integration/Whitespace.pm
index 54db69acb..7eb2864f8 100644
--- a/perllib/Open311/Endpoint/Integration/Whitespace.pm
+++ b/perllib/Open311/Endpoint/Integration/Whitespace.pm
@@ -79,6 +79,7 @@ sub post_service_request {
     my $integration = $self->get_integration;
 
     $args->{attributes}{location_of_containers} //= '';
+    $args->{attributes}{location_of_letterbox} //= '';
     $args->{attributes}{quantity} ||= 1;
 
     my $worksheet_id = $integration->CreateWorksheet({
@@ -89,6 +90,7 @@ sub post_service_request {
         worksheet_message => $self->_worksheet_message($args),
         assisted_yn => $args->{attributes}->{assisted_yn},
         location_of_containers => $args->{attributes}->{location_of_containers},
+        location_of_letterbox => $args->{attributes}->{location_of_letterbox},
         quantity => $args->{attributes}->{quantity},
     });
 
diff --git a/perllib/Open311/Endpoint/Service/UKCouncil/Whitespace.pm b/perllib/Open311/Endpoint/Service/UKCouncil/Whitespace.pm
index 3ebf1e5fc..fe1f10833 100644
--- a/perllib/Open311/Endpoint/Service/UKCouncil/Whitespace.pm
+++ b/perllib/Open311/Endpoint/Service/UKCouncil/Whitespace.pm
@@ -46,6 +46,13 @@ sub _build_attributes {
             required => 0,
             automated => 'hidden_field',
         ),
+        Open311::Endpoint::Service::Attribute->new(
+            code => 'location_of_letterbox',
+            description => 'Location of letterbox',
+            datatype => 'string',
+            required => 0,
+            automated => 'hidden_field',
+        ),
         Open311::Endpoint::Service::Attribute->new(
             code => 'quantity',
             description => 'Number of containers',
diff --git a/t/open311/endpoint/whitespace.t b/t/open311/endpoint/whitespace.t
index 4ddb29372..6588c4cc3 100644
--- a/t/open311/endpoint/whitespace.t
+++ b/t/open311/endpoint/whitespace.t
@@ -109,7 +109,8 @@ subtest "GET service" => sub {
           { code => 'fixmystreet_id', order => 3, required => 'true', variable => 'false', datatype => 'string', datatype_description => '', automated => 'server_set', description => 'external system ID' },
           { code => 'assisted_yn', order => 4, required => 'false', variable => 'true', datatype => 'string', datatype_description => '', automated => 'hidden_field', description => 'Assisted collection (Yes/No)' },
           { code => 'location_of_containers', order => 5, required => 'false', variable => 'true', datatype => 'string', datatype_description => '', automated => 'hidden_field', description => 'Location of containers' },
-          { code => 'quantity', order => 6, required => 'false', variable => 'true', datatype => 'string', datatype_description => '', automated => 'hidden_field', description => 'Number of containers' },
+          { code => 'location_of_letterbox', order => 6, required => 'false', variable => 'true', datatype => 'string', datatype_description => '', automated => 'hidden_field', description => 'Location of letterbox' },
+          { code => 'quantity', order => 7, required => 'false', variable => 'true', datatype => 'string', datatype_description => '', automated => 'hidden_field', description => 'Number of containers' },
       ],
     }, 'correct json returned';
 };

From 811214941bc7a05d5c536d0a81ca36d29528630b Mon Sep 17 00:00:00 2001
From: Nik Gupta <vngupta77@gmail.com>
Date: Tue, 3 Dec 2024 13:23:55 +0000
Subject: [PATCH 18/23] [Confirm] Log GraphQL errors.

---
 perllib/Integrations/Confirm.pm |  7 +++++-
 t/integrations/confirm.t        | 43 +++++++++++++++++++++++++++++++++
 t/integrations/confirm.yml      |  2 ++
 3 files changed, 51 insertions(+), 1 deletion(-)
 create mode 100644 t/integrations/confirm.t
 create mode 100644 t/integrations/confirm.yml

diff --git a/perllib/Integrations/Confirm.pm b/perllib/Integrations/Confirm.pm
index 1d91cfc45..c824c3d26 100644
--- a/perllib/Integrations/Confirm.pm
+++ b/perllib/Integrations/Confirm.pm
@@ -315,13 +315,18 @@ sub perform_request_graphql {
     my $body = {
         query => $query,
     };
+    my $encoded_body = encode_json($body);
 
-    $request->content(encode_json($body));
+    $request->content($encoded_body);
 
     my $response = $self->ua->request($request);
 
     my $content = decode_json($response->content);
 
+    if ($content->{errors} && @{$content->{errors}}) {
+        $self->logger->warn("Got errors in response to GraphQL query $encoded_body: " . $response->content);
+    }
+
     return $content;
 }
 
diff --git a/t/integrations/confirm.t b/t/integrations/confirm.t
new file mode 100644
index 000000000..1888cbff1
--- /dev/null
+++ b/t/integrations/confirm.t
@@ -0,0 +1,43 @@
+package Integrations::Confirm::Dummy;
+use Path::Tiny;
+use Moo;
+extends 'Integrations::Confirm';
+sub _build_config_file { path(__FILE__)->sibling("confirm.yml")->stringify }
+
+package main;
+
+use strict;
+use warnings;
+
+use Test::More;
+use Test::MockModule;
+use Test::Output;
+use HTTP::Response;
+use JSON::MaybeXS;
+
+BEGIN { $ENV{TEST_MODE} = 1; }
+
+my $lwp = Test::MockModule->new('LWP::UserAgent');
+
+$lwp->mock(request => sub {
+    return HTTP::Response->new(200, 'OK', [], encode_json({
+            data => {},
+            errors => [
+                {
+                    message => "Uh-oh spaghettio!",
+                }
+            ]
+        })
+    );
+});
+
+my $integration = Integrations::Confirm::Dummy->new;
+
+subtest "GraphQL errors are logged " => sub {
+    local $ENV{TEST_LOGGER} = 'warn';
+    stderr_like {
+        $integration->perform_request_graphql(type => 'job_types');
+    } qr/.*Uh-oh spaghettio!.*/, 'Got expected warning log for errors.';
+};
+
+done_testing;
diff --git a/t/integrations/confirm.yml b/t/integrations/confirm.yml
new file mode 100644
index 000000000..7bf19bd91
--- /dev/null
+++ b/t/integrations/confirm.yml
@@ -0,0 +1,2 @@
+graphql_url: "http://example.org/graphql"
+graphql_key: "key"

From 3954ecab6b831f29912c31dca089d0cb13b6bab4 Mon Sep 17 00:00:00 2001
From: Matthew Somerville <matthew@mysociety.org>
Date: Tue, 10 Dec 2024 12:37:52 +0000
Subject: [PATCH 19/23] [Whitespace] Make sure SOAP 1.1 is used.

See e.g. 1e9f8f0e9 for Echo.
---
 perllib/Integrations/Whitespace.pm | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/perllib/Integrations/Whitespace.pm b/perllib/Integrations/Whitespace.pm
index 03f2d1c43..1ccd57bd1 100644
--- a/perllib/Integrations/Whitespace.pm
+++ b/perllib/Integrations/Whitespace.pm
@@ -33,7 +33,6 @@ has endpoint => (
         $ENV{PERL_LWP_SSL_CA_PATH} = '/etc/ssl/certs' unless $ENV{DEV_USE_SYSTEM_CA_PATH};
 
         my $soap = SOAP::Lite->new(
-            soapversion => 1.1,
             proxy => $self->url,
             default_ns => $self->attr,
             on_action => sub { $self->attr . $_[1] }
@@ -67,6 +66,8 @@ sub call {
 
     require SOAP::Lite;
     @params = make_soap_structure(@params);
+    # See comment in Echo.pm
+    SOAP::Lite->soapversion(1.1);
     my $som = $self->endpoint->call(
         $method => @params,
         $self->security

From 739c9386874d2280cb61a6d6009c4eb6ceddb222 Mon Sep 17 00:00:00 2001
From: Matthew Somerville <matthew@mysociety.org>
Date: Wed, 31 Jul 2024 14:21:10 +0100
Subject: [PATCH 20/23] [SLWP] Add way to pass in multiple payment details.

---
 perllib/Open311/Endpoint/Integration/Echo.pm  | 30 +++++++++
 .../Endpoint/Integration/UK/Merton/Echo.pm    | 66 +------------------
 perllib/Open311/Endpoint/Role/SLWP.pm         | 27 ++++----
 t/open311/endpoint/merton_echo.t              |  1 +
 4 files changed, 43 insertions(+), 81 deletions(-)

diff --git a/perllib/Open311/Endpoint/Integration/Echo.pm b/perllib/Open311/Endpoint/Integration/Echo.pm
index 5ff8b7a10..748eb65d0 100644
--- a/perllib/Open311/Endpoint/Integration/Echo.pm
+++ b/perllib/Open311/Endpoint/Integration/Echo.pm
@@ -13,6 +13,7 @@ pushed to us by Echo or fetched directly.
 package Open311::Endpoint::Integration::Echo;
 
 use v5.14;
+use utf8;
 
 use Moo;
 use JSON::MaybeXS;
@@ -501,4 +502,33 @@ used as part of a Multi integration.
 
 sub get_service_request_updates { }
 
+=head2 update_event_payment
+
+Given an update args and an arrayref of payment details, update the event
+to add those details. This is currently only for SLWP, and includes
+hard-coded SLWP IDs.
+
+=cut
+
+sub update_event_payment {
+    my ($self, $args, $payments) = @_;
+
+    my $integ = $self->get_integration;
+    my $event = $integ->GetEvent($args->{service_request_id});
+    my $data = [];
+    foreach (@$payments) {
+        $_->{amount} =~ s/£//;
+        push @$data, {
+            # Could GetEventType and loop through it all to find these IDs out but for just this seemed okay
+            id => 27409,
+            childdata => [
+                { id => 27410, value => $_->{ref} },
+                { id => 27411, value => $_->{amount} },
+            ],
+        };
+    }
+    $integ->UpdateEvent({ id => $event->{Id}, data => $data });
+    $args->{description} = ''; # Blank out so nothing sent to Echo now
+}
+
 1;
diff --git a/perllib/Open311/Endpoint/Integration/UK/Merton/Echo.pm b/perllib/Open311/Endpoint/Integration/UK/Merton/Echo.pm
index 7f63396fa..4ccc6e228 100644
--- a/perllib/Open311/Endpoint/Integration/UK/Merton/Echo.pm
+++ b/perllib/Open311/Endpoint/Integration/UK/Merton/Echo.pm
@@ -13,9 +13,9 @@ Merton specifics for its Echo backend
 package Open311::Endpoint::Integration::UK::Merton::Echo;
 
 use utf8;
-use DateTime;
 use Moo;
 extends 'Open311::Endpoint::Integration::Echo';
+with 'Open311::Endpoint::Role::SLWP';
 
 around BUILDARGS => sub {
     my ($orig, $class, %args) = @_;
@@ -42,68 +42,4 @@ around process_service_request_args => sub {
     return $request;
 };
 
-around check_for_data_value => sub {
-    my ($orig, $class, $name, $args, $request, $parent_name) = @_;
-
-    my $service = $args->{attributes}{service_id} || '';
-
-    return 1 if $name eq 'Container Mix' && ($service eq '2241' || $service eq '2250' || $service eq '2246' || $service eq '3571');
-    return 1 if $name eq 'Paper' && ($service eq '2240' || $service eq '2249' || $service eq '2632');
-    return 1 if $name eq 'Food' && ($service eq '2239' || $service eq '2248');
-    return 1 if ($name eq 'Garden' || $name eq 'Garden Waste ') && $service eq '2247'; # Attribute has a space at the end
-    return 1 if $name eq 'Refuse Bag' && $service eq '2242';
-    return 1 if $name eq 'Refuse Bin' && ($service eq '2238' || $service eq '2243' || $service eq '3576');
-
-    # Garden waste
-    if ($args->{service_code} eq '1638') {
-        my $method = $args->{attributes}{LastPayMethod} || '';
-        return 2 if $name eq 'Payment Type' && $method eq 3; # DD
-        return 3 if $name eq 'Payment Type' && $method eq 4; # 'cheque' (or phone)
-    }
-
-    # Bulky items
-    if ($args->{service_code} eq '1636') {
-        # Default in configuration is Payment Type 1 (Card), Payment Method 2 (Website)
-        my $method = $args->{attributes}{payment_method} || '';
-        return 2 if $name eq 'Payment Type' && $method eq 'cheque'; # Cheque
-        if ($name eq 'Payment Method') {
-            return 1 if $method eq 'csc' || $method eq 'cheque'; # Borough Phone Payment
-            return 2 if $method eq 'credit_card'; # Borough Website Payment
-        }
-    }
-
-    return $class->$orig($name, $args, $request, $parent_name);
-};
-
-around post_service_request_update => sub {
-    my ($orig, $class, $args) = @_;
-    return $class->$orig($args) unless $args->{description};
-
-    if ($args->{description} =~ /Payment confirmed, reference (.*), amount (.*)/) {
-        my ($ref, $amount) = ($1, $2);
-        $amount =~ s/£//;
-        my $integ = $class->get_integration;
-        my $event = $integ->GetEvent($args->{service_request_id});
-        # Could GetEventType and loop through it all to find these IDs out but for just this seemed okay
-        my $data = {
-            id => 27409,
-            childdata => [
-                { id => 27410, value => $ref },
-                { id => 27411, value => $amount },
-            ],
-        };
-        $integ->UpdateEvent({ id => $event->{Id}, data => [ $data ] });
-        $args->{description} = ''; # Blank out so nothing sent to Echo now
-    }
-
-    if ($args->{description} =~ /Booking cancelled/) {
-        $args->{actiontype_id} = 8;
-        $args->{datatype_id} = 0;
-    }
-
-    my $result = $class->$orig($args);
-
-    return $result;
-};
-
 1;
diff --git a/perllib/Open311/Endpoint/Role/SLWP.pm b/perllib/Open311/Endpoint/Role/SLWP.pm
index 5970f2146..20e9a8680 100644
--- a/perllib/Open311/Endpoint/Role/SLWP.pm
+++ b/perllib/Open311/Endpoint/Role/SLWP.pm
@@ -23,7 +23,7 @@ around check_for_data_value => sub {
     return 1 if $name eq 'Container Mix' && ($service eq '2241' || $service eq '2250' || $service eq '2246' || $service eq '3571');
     return 1 if $name eq 'Paper' && ($service eq '2240' || $service eq '2249' || $service eq '2632');
     return 1 if $name eq 'Food' && ($service eq '2239' || $service eq '2248');
-    return 1 if $name eq 'Garden' && $service eq '2247';
+    return 1 if ($name eq 'Garden' || $name eq 'Garden Waste ') && $service eq '2247'; # Attribute has a space at the end
     return 1 if $name eq 'Refuse Bag' && $service eq '2242';
     return 1 if $name eq 'Refuse Bin' && ($service eq '2238' || $service eq '2243' || $service eq '3576');
 
@@ -45,24 +45,19 @@ around post_service_request_update => sub {
     my ($orig, $class, $args) = @_;
     return $class->$orig($args) unless $args->{description};
 
-    if ($args->{description} =~ /Payment confirmed, reference (.*), amount (.*)/) {
+    if (my $payments = $args->{attributes}{payments}) {
+        my @data = split /\|/, $payments;
+        my @payments;
+        for (my $i=0; $i<@data; $i+=2) {
+            push @payments, { ref => $data[$i], amount => $data[$i+1] }
+        }
+        $class->update_event_payment($args, \@payments);
+    } elsif ($args->{description} =~ /Payment confirmed, reference (.*), amount (.*)/) {
         my ($ref, $amount) = ($1, $2);
-        $amount =~ s/£//;
-        my $integ = $class->get_integration;
-        my $event = $integ->GetEvent($args->{service_request_id});
-        # Could GetEventType and loop through it all to find these IDs out but for just this seemed okay
-        my $data = {
-            id => 27409,
-            childdata => [
-                { id => 27410, value => $ref },
-                { id => 27411, value => $amount },
-            ],
-        };
-        $integ->UpdateEvent({ id => $event->{Id}, data => [ $data ] });
-        $args->{description} = ''; # Blank out so nothing sent to Echo now
+        $class->update_event_payment($args, [ { ref => $ref, amount => $amount } ]);
     }
 
-    if ($args->{description} eq 'Booking cancelled by customer') {
+    if ($args->{description} =~ /Booking cancelled/ || $args->{attributes}{booking_cancelled}) {
         $args->{actiontype_id} = 8;
         $args->{datatype_id} = 0;
     }
diff --git a/t/open311/endpoint/merton_echo.t b/t/open311/endpoint/merton_echo.t
index a55aaf100..c3c2b38be 100644
--- a/t/open311/endpoint/merton_echo.t
+++ b/t/open311/endpoint/merton_echo.t
@@ -268,6 +268,7 @@ subtest "POST a successful payment" => sub {
         update_id => 456,
         status => 'OPEN',
         description => 'Payment confirmed, reference ABC, amount £34.56',
+        'attribute[payments]' => 'ABC|34.56',
         first_name => 'Bob',
         last_name => 'Mould',
     );

From b8e093ddafe4f1b6d217cad92b6b61b5c11a44aa Mon Sep 17 00:00:00 2001
From: Matthew Somerville <matthew@mysociety.org>
Date: Wed, 11 Dec 2024 20:23:33 +0000
Subject: [PATCH 21/23] [Confirm] Cope if enquiry JSON returns error.

---
 perllib/Open311/Endpoint/Integration/Confirm.pm |  3 ++-
 t/open311/endpoint/lincolnshire.t               | 15 +++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/perllib/Open311/Endpoint/Integration/Confirm.pm b/perllib/Open311/Endpoint/Integration/Confirm.pm
index 78beaf522..39a8ad6c7 100644
--- a/perllib/Open311/Endpoint/Integration/Confirm.pm
+++ b/perllib/Open311/Endpoint/Integration/Confirm.pm
@@ -638,7 +638,8 @@ sub photo_urls_for_update {
     my ($self, $enquiry_id) = @_;
     my $integ = $self->get_integration;
 
-    my $job_id = $integ->get_enquiry_json($enquiry_id)->{primaryJobNumber} or return;
+    my $enquiry = $integ->get_enquiry_json($enquiry_id) or return;
+    my $job_id = $enquiry->{primaryJobNumber};
     my $documents = $integ->documents_for_job($job_id) or return;
 
     my @ids = map { $_->{documentNo} } grep { $self->photo_filter($_) } @$documents;
diff --git a/t/open311/endpoint/lincolnshire.t b/t/open311/endpoint/lincolnshire.t
index af50d3a8d..a0a50517e 100644
--- a/t/open311/endpoint/lincolnshire.t
+++ b/t/open311/endpoint/lincolnshire.t
@@ -55,6 +55,21 @@ $open311->mock(perform_request => sub {
     return {};
 });
 
+subtest "bad look up of completion photo" => sub {
+    my $endpoint = Open311::Endpoint::Integration::UK::Dummy->new;
+    my $lwp = Test::MockModule->new('LWP::UserAgent');
+    $lwp->mock(request => sub {
+        my ($ua, $req) = @_;
+        return HTTP::Response->new(200, 'OK', [], '{"access_token":"123","expires_in":3600}') if $req->uri =~ /oauth\/token/;
+        return HTTP::Response->new(404, 'Not found', [], '<html><body>Hmm</body></html>') if $req->uri =~ /enquiries\/2020/;
+    });
+    my $res = $endpoint->run_test_request(
+        GET => '/servicerequestupdates.xml?start_date=2022-10-23T00:00:00Z&end_date=2022-10-24T00:00:00Z',
+    );
+    ok $res->is_success, 'valid request' or diag $res->content;
+    contains_string $res->content, '<media_url></media_url>';
+};
+
 subtest "looking up of completion photos" => sub {
     my $endpoint = Open311::Endpoint::Integration::UK::Dummy->new;
     my $lwp = Test::MockModule->new('LWP::UserAgent');

From 881550c70b9358cfd2771a22d4468a52b504aa95 Mon Sep 17 00:00:00 2001
From: Matthew Somerville <matthew@mysociety.org>
Date: Fri, 6 Dec 2024 13:12:35 +0000
Subject: [PATCH 22/23] [Brent] Include car park issue type in notes.

---
 perllib/Open311/Endpoint/Integration/UK/Brent/Echo.pm | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/perllib/Open311/Endpoint/Integration/UK/Brent/Echo.pm b/perllib/Open311/Endpoint/Integration/UK/Brent/Echo.pm
index f91390461..8f901c7e7 100644
--- a/perllib/Open311/Endpoint/Integration/UK/Brent/Echo.pm
+++ b/perllib/Open311/Endpoint/Integration/UK/Brent/Echo.pm
@@ -13,6 +13,10 @@ around process_service_request_args => sub {
     my ($orig, $class, $args) = @_;
     my $request = $class->$orig($args);
 
+    if ($args->{attributes}{CarParkIssueType}) {
+        $args->{attributes}{description} .= " | " . $args->{attributes}{CarParkIssueType};
+    }
+
     # Missed collection
     if ($request->{event_type} == 2891) {
         my $service_id = $args->{attributes}{service_id};

From 6f47809a42aa9ab31b9b1a371417bf3151f54e10 Mon Sep 17 00:00:00 2001
From: Moray Jones <moray@mysociety.org>
Date: Fri, 20 Dec 2024 09:30:54 +0000
Subject: [PATCH 23/23] [Bristol] Convert Bristol to a Multi

Bristol has a passthrough which needs to continue
to work alongsider the new Alloy integration.

https://github.com/mysociety/societyworks/issues/4683
---
 conf/council-bristol_alloy.yml-example        | 57 +++++++++++++++++++
 conf/council-www.bristol.gov.uk.yml-example   |  2 +
 .../Endpoint/Integration/UK/Bristol.pm        | 31 ++++++++++
 .../Endpoint/Integration/UK/Bristol/Alloy.pm  | 20 +++++++
 .../Integration/UK/Bristol/Passthrough.pm     | 23 ++++++++
 t/open311/endpoint/uk.t                       |  5 ++
 6 files changed, 138 insertions(+)
 create mode 100644 conf/council-bristol_alloy.yml-example
 create mode 100644 conf/council-www.bristol.gov.uk.yml-example
 create mode 100644 perllib/Open311/Endpoint/Integration/UK/Bristol.pm
 create mode 100644 perllib/Open311/Endpoint/Integration/UK/Bristol/Alloy.pm
 create mode 100644 perllib/Open311/Endpoint/Integration/UK/Bristol/Passthrough.pm

diff --git a/conf/council-bristol_alloy.yml-example b/conf/council-bristol_alloy.yml-example
new file mode 100644
index 000000000..fd484d208
--- /dev/null
+++ b/conf/council-bristol_alloy.yml-example
@@ -0,0 +1,57 @@
+# Currently just a copy of Bucks
+{
+  "api_key": "",
+  "api_url": "",
+  "rfs_design": '',
+  "resource_attachment_attribute_id": "",
+
+  "category_list_code": "",
+  "category_title_attribute": "",
+
+  "parent_attribute_name": "",
+
+  "service_whitelist": {
+  },
+
+  "resource_attribute_defaults": {
+  },
+
+  "request_to_resource_attribute_mapping": {
+  },
+  "request_to_resource_attribute_manual_mapping": {
+  },
+
+  "created_datetime_attribute_id": '',
+
+  "inspection_attribute_mapping": {
+  },
+
+  "inspection_status_mapping": {
+  },
+
+  "inspection_closure_mapping": {
+  },
+
+  "ignored_defect_types": [
+  ],
+
+  "defect_sourcetype_category_mapping": {
+  },
+
+  "defect_status_mapping": {
+  },
+
+  "defect_resource_name": [],
+
+  "defect_attribute_mapping": {
+  },
+
+  "contact": {
+      "code": "",
+      "attribute_id": "",
+      "attribute_defaults": {
+      },
+      "attribute_mapping": {
+      },
+  }
+}
diff --git a/conf/council-www.bristol.gov.uk.yml-example b/conf/council-www.bristol.gov.uk.yml-example
new file mode 100644
index 000000000..2c16d9f4d
--- /dev/null
+++ b/conf/council-www.bristol.gov.uk.yml-example
@@ -0,0 +1,2 @@
+endpoint: https://localhost:4000/
+api_key: 123
diff --git a/perllib/Open311/Endpoint/Integration/UK/Bristol.pm b/perllib/Open311/Endpoint/Integration/UK/Bristol.pm
new file mode 100644
index 000000000..a280f84b7
--- /dev/null
+++ b/perllib/Open311/Endpoint/Integration/UK/Bristol.pm
@@ -0,0 +1,31 @@
+=head1 NAME
+
+Open311::Endpoint::Integration::UK::Bristol - Bristol integration set-up
+
+=head1 SYNOPSIS
+
+Bristol manage their own Open311 server, but have a managed Alloy integration, so is
+set up as a Multi integration with Alloy and a Passthrough
+
+=cut
+
+package Open311::Endpoint::Integration::UK::Bristol;
+
+use Moo;
+extends 'Open311::Endpoint::Integration::Multi';
+
+use Module::Pluggable
+    search_path => ['Open311::Endpoint::Integration::UK::Bristol'],
+    instantiate => 'new';
+
+has jurisdiction_id => (
+    is => 'ro',
+    default => 'bristol',
+);
+
+has integration_without_prefix => (
+    is => 'ro',
+    default => 'Passthrough',
+);
+
+__PACKAGE__->run_if_script;
diff --git a/perllib/Open311/Endpoint/Integration/UK/Bristol/Alloy.pm b/perllib/Open311/Endpoint/Integration/UK/Bristol/Alloy.pm
new file mode 100644
index 000000000..161806d21
--- /dev/null
+++ b/perllib/Open311/Endpoint/Integration/UK/Bristol/Alloy.pm
@@ -0,0 +1,20 @@
+=head1 NAME
+
+Open311::Endpoint::Integration::UK::Bristol::Alloy - Bristol-specific parts of its Alloy integration
+
+=head1 DESCRIPTION
+
+=cut
+
+package Open311::Endpoint::Integration::UK::Bristol::Alloy;
+
+use Moo;
+extends 'Open311::Endpoint::Integration::AlloyV2';
+
+around BUILDARGS => sub {
+    my ($orig, $class, %args) = @_;
+    $args{jurisdiction_id} = 'bristol_alloy';
+    return $class->$orig(%args);
+};
+
+1;
diff --git a/perllib/Open311/Endpoint/Integration/UK/Bristol/Passthrough.pm b/perllib/Open311/Endpoint/Integration/UK/Bristol/Passthrough.pm
new file mode 100644
index 000000000..081d05a41
--- /dev/null
+++ b/perllib/Open311/Endpoint/Integration/UK/Bristol/Passthrough.pm
@@ -0,0 +1,23 @@
+=head1 NAME
+
+Open311::Endpoint::Integration::UK::Bristol::Passthrough - Bristol Passthrough backend
+
+=head1 SUMMARY
+
+This is the Bristol-specific Passthrough integration. It is a standard
+Open311 server.
+
+=cut
+
+package Open311::Endpoint::Integration::UK::Bristol::Passthrough;
+
+use Moo;
+extends 'Open311::Endpoint::Integration::Passthrough';
+
+around BUILDARGS => sub {
+    my ($orig, $class, %args) = @_;
+    $args{jurisdiction_id} = 'www.bristol.gov.uk';
+    return $class->$orig(%args);
+};
+
+1;
diff --git a/t/open311/endpoint/uk.t b/t/open311/endpoint/uk.t
index 3920934f4..f8a704059 100644
--- a/t/open311/endpoint/uk.t
+++ b/t/open311/endpoint/uk.t
@@ -47,6 +47,11 @@ test_multi(0, 'Open311::Endpoint::Integration::UK::Merton',
     #'Open311::Endpoint::Integration::UK::Merton::Passthrough' => 'www.merton.gov.uk',
 );
 
+test_multi(0, 'Open311::Endpoint::Integration::UK::Bristol',
+    'Open311::Endpoint::Integration::UK::Bristol::Alloy' => 'bristol_alloy',
+    #'Open311::Endpoint::Integration::UK::Bristol::Passthrough' => 'www.bristol.gov.uk',
+);
+
 test_multi(1, 'Open311::Endpoint::Integration::UK::Peterborough',
     'Open311::Endpoint::Integration::UK::Peterborough::Confirm' => 'peterborough_confirm',
     'Open311::Endpoint::Integration::UK::Peterborough::Ezytreev' => 'peterborough_ezytreev',