Add: support relations in split_solve & partitions

LINKING_RELATIONS := Relations that link multiple services to be on the
	same route (
		order, same_route, sequence, shipment
	) are supported in split_solve algorithm (`max_split_size`) and
	partitions (`[:configuration][:preprocessing][:partitions]`)

FORCING_RELATIONS := Relations that force multiple services/vehicle to
	stay in the same VRP (
		vehicle_trips, meetup,
		minimum_duration_lapse, maximum_duration_lapse,
		minimum_day_lapse, maximum_day_lapse
	) are respected by split_solve (`max_split_size`) algorithm

CHANGELOG.md

@@ -6,6 +6,8 @@
 
 - Corresponding vehicle_id is returned within each service's skills if problem is partitioned with vehicle entity [#110](https://github.com/Mapotempo/optimizer-api/pull/110)
 - Support initial routes and skills in split_solve (`max_split_size`) algorithm [#140](https://github.com/Mapotempo/optimizer-api/pull/140)
+- Support relations (`order`, `same_route`, `sequence`, `shipment`) in split_solve algorithm (`max_split_size`) and partitions (`[:configuration][:preprocessing][:partitions]`) [](https://github.com/Mapotempo/optimizer-api/pull/)
+- split_solve algorithm (`max_split_size`) respects relations (`vehicle_trips`, `meetup`, `minimum_duration_lapse`, `maximum_duration_lapse`, `minimum_day_lapse`, `maximum_day_lapse`) [](https://github.com/Mapotempo/optimizer-api/pull/)
 
 ### Changed
 

Gemfile

@@ -30,7 +30,7 @@ gem 'resque-status', '>0.4'
 gem 'rest-client'
 
 gem 'ai4r'
-gem 'balanced_vrp_clustering', github: 'Mapotempo/balanced_vrp_clustering', branch: 'dev'
+gem 'balanced_vrp_clustering', github: 'senhalil/balanced_vrp_clustering', branch: 'dev' # Replace senhalil with Mapotempo when the following PR is merged https://github.com/Mapotempo/balanced_vrp_clustering/pull/16
 gem 'sim_annealing'
 
 gem 'polylines'

Gemfile.lock

@@ -7,11 +7,11 @@ GIT
  activesupport (>= 5.0.0)
 
 GIT
- remote: https://github.com/Mapotempo/balanced_vrp_clustering.git
- revision: f05b84f49cfc8803ef0adc2bdc12627720d82796
+ remote: https://github.com/senhalil/balanced_vrp_clustering.git
+ revision: 657b71af8721cbbae44c5bba36790fb1c1d6842f
  branch: dev
  specs:
- balanced_vrp_clustering (0.1.7)
+ balanced_vrp_clustering (0.2.0)
  awesome_print
  color-generator
  geojson2image
@@ -52,7 +52,7 @@ GEM
  anyway_config (2.0.6)
  ruby-next-core (>= 0.8.0)
  ast (2.4.1)
- awesome_print (1.8.0)
+ awesome_print (1.9.2)
  backport (1.1.2)
  benchmark (0.1.0)
  benchmark-ips (2.8.3)

lib/heuristics/dichotomious_approach.rb

@@ -502,7 +502,7 @@ def self.kmeans(vrp, cut_symbol)
 
  options[:clusters_infos] = SplitClustering.collect_cluster_data(vrp, nb_clusters)
 
- clusters = SplitClustering.kmeans_process(nb_clusters, data_items, unit_symbols, limits, options)
+ clusters = SplitClustering.kmeans_process(nb_clusters, data_items, {}, limits, options)
 
  services_by_cluster = clusters.collect{ |cluster|
  cluster.data_items.flat_map{ |data|

lib/interpreters/split_clustering.rb

@@ -28,13 +28,22 @@
 
 module Interpreters
  class SplitClustering
- # Relations that link multiple services to on the same route
+ # Relations that link multiple services to be on the same route
  LINKING_RELATIONS = %i[
  order
  same_route
  sequence
  shipment
  ].freeze
+ # Relations that force multiple services/vehicles to stay in the same VRP
+ FORCING_RELATIONS = %i[
+ maximum_day_lapse
+ maximum_duration_lapse
+ meetup
+ minimum_day_lapse
+ minimum_duration_lapse
+ vehicle_trips
+ ].freeze
 
  # TODO: private method
  def self.split_clusters(service_vrp, job = nil, &block)
@@ -141,17 +150,24 @@ def self.split_solve_candidate?(service_vrp)
  vrp.shipments.empty? # Clustering supports Shipment only as Relation TODO: delete this check when Model::Shipment is removed
  else
  ss_data = service_vrp[:split_solve_data]
- current_vehicles = ss_data[:current_vehicles]
- service_vehicle_assignments = ss_data[:service_vehicle_assignments] # no empties_or_fills in here
+ return false if ss_data[:cannot_split_further]
+
+ # filter the vehicles that are forced to be on the same side
+ current_vehicle_ids = ss_data[:current_vehicles].map(&:id)
+ ss_data[:representative_vrp].relations.each{ |rel|
+ raise 'There should be only :same_route relations inside the representative_vrp.' if rel.type != :same_route
+
+ current_vehicle_ids << rel.linked_ids.join if current_vehicle_ids.reject!{ |id| rel.linked_ids.include?(id) }
+ }
 
- current_vehicles.size > 1 &&
- (ss_data[:current_vehicle_limit].nil? || ss_data[:current_vehicle_limit] > 1) &&
- current_vehicles.sum{ |v| service_vehicle_assignments[v.id].size } > vrp.preprocessing_max_split_size
+ current_vehicle_ids.size > 1 && (ss_data[:current_vehicle_limit] || Helper.fixnum_max) > 1 &&
+ ss_data[:current_vehicles].sum{ |vehicle|
+ ss_data[:service_vehicle_assignments][vehicle.id].size
+ } > vrp.preprocessing_max_split_size
  end
  end
 
- # TODO: 0- see below, there are multiple
- # TODO: 1- implement a notion of "same_vehicle" relation inside balanced_vrp_clustering gem
+ # TODO: Following are ways to improve split_solve performance (2, 3, 4 cpu time) (5 solution quality)
  # TODO: 2- decrease iteration-complexity by "relaxing" the convergence limits (movement, balance-violation) of k-means for max_split?
  # TODO: 3- decrease point-complexity by "grouping" by lat/lon more aggressively for this split
  # TODO: 4- decrease vehicle-complexity by improving balanced_vrp_clustering
@@ -213,10 +229,18 @@ def self.split_solve_core(service_vrp, job = nil, &block)
  sides = split_balanced_kmeans(
  { vrp: create_representative_sub_vrp(ss_data) }, 2,
  cut_symbol: :duration, restarts: 3, build_sub_vrps: false, basic_split: true, group_points: false
- ).sort_by!{ |side| [side.size, side.sum(&:visits_number)] }.reverse!
- sides.collect!{ |side| enum_current_vehicles.select{ |v| side.any?{ |s| s.id == v.id } } }
+ ).sort_by!{ |side|
+ [side.size, side.sum(&:visits_number)] # [number_of_vehicles, number_of_visits]
+ }.reverse!.collect!{ |side|
+ enum_current_vehicles.select{ |v| side.any?{ |s| s.id == v.id } }
+ }
 
- log 'There should be exactly two clusters in split_solve_core!', level: :warn unless sides.size == 2 && sides.none?(&:empty?)
+ unless sides.size == 2 && sides.none?(&:empty?)
+ # this might happen under certain cases (skills etc can force all points to be on one side)
+ # and not necessarily a problem but it should happen very rarely (in real instances)
+ log 'There should be exactly two clusters in split_solve_core!', level: :warn
+ ss_data[:cannot_split_further] = true
+ end
 
  split_service_counts = sides.collect{ |current_vehicles|
  current_vehicles.sum{ |v| ss_data[:service_vehicle_assignments][v.id].size }
@@ -229,13 +253,13 @@ def self.split_solve_core(service_vrp, job = nil, &block)
 
  ss_data[:current_vehicles] = side
 
- vehicle_limit_ratio = current_vehicle_limit.to_f * side.size / enum_current_vehicles.size
+ v_limit = current_vehicle_limit.to_f * side.size / enum_current_vehicles.size
  # Warning: round does not work if there is an even "half" split
- ss_data[:current_vehicle_limit] = current_vehicle_limit &&
- (index.zero? ? vehicle_limit_ratio.ceil : vehicle_limit_ratio.floor)
+ ss_data[:current_vehicle_limit] = current_vehicle_limit && (index.zero? ? v_limit.ceil : v_limit.floor)
 
  split_solve_core(service_vrp, job = nil, &block)
  }
+ ss_data[:cannot_split_further] = false
  log "<-- split_solve_core level: #{split_level}"
 
  Helper.merge_results(results)
@@ -307,24 +331,16 @@ def self.create_sub_vrp(split_solve_data)
  end
 
  def self.create_representative_vrp(split_solve_data)
- # This VRP represent the original VRP only `m` number of points by reducing the services belonging to the
+ # This VRP represent the original VRP with only `m` number of points by reducing the services belonging to the
  # same vehicle-zone to a single point (with average lat/lon and total duration/visits). Where `m` is the
- # number of non-empty vehicle-zones coming from the very first split_by_vehicle.
+ # number of non-empty vehicle-zones generated by the very first split_by_vehicle.
+
  points = []
  services = []
- # TODO: 0- relations needs to be taken into account inside clustering during this split
- # - the vehicles need to be in the same sub-vrp for the following relations:
- # => vehicle_trips
- # - the services need to be in the same sub-vrp for the following relations:
- # => meetup, minimum_duration_lapse, maximum_duration_lapse, minimum_day_lapse, maximum_day_lapse
- # (we need to go thorugh the original relations and "connect" the "vehicle_id"s below with "same_route")
- relations = []
-
  split_solve_data[:service_vehicle_assignments].each{ |vehicle_id, vehicle_services|
- # TODO: After relations are taken into account inside clustering, we don't have to
- # decrease the number of points to 1. We can represent each group with multiple
+ # TODO: We don't have to represent each group with only 1 point. We can represent each group with multiple
  # points, carefully selected to represent the mean, median and extremes of the group
- # and "relate" these points so that they will stay on the same "side" in the 2-split
+ # and "relate" these points with :same_route so that they will stay on the same "side" in the 2-split
  average_lat = vehicle_services.sum{ |s| s.activity.point.location.lat } / vehicle_services.size.to_f
  average_lon = vehicle_services.sum{ |s| s.activity.point.location.lon } / vehicle_services.size.to_f
  points << { id: "p#{vehicle_id}", location: { lat: average_lat, lon: average_lon }}
@@ -338,6 +354,26 @@ def self.create_representative_vrp(split_solve_data)
  }
  }
 
+ relations = []
+ # go thorugh the original relations and force the services and vehicles to stay in the same sub-vrp if necessary
+ split_solve_data[:original_vrp].relations.select{ |r| FORCING_RELATIONS.include?(r.type) }.each{ |relation|
+ if relation.linked_vehicle_ids.any? && relation.linked_services.none?
+ relations << { type: :same_route, linked_ids: relation.linked_vehicle_ids }
+ elsif relation.linked_vehicle_ids.none? && relation.linked_services.any?
+ linked_ids = []
+ relation.linked_services.each{ |linked_service|
+ split_solve_data[:service_vehicle_assignments].any?{ |v_id, v_services|
+ linked_ids << v_id if v_services.include?(linked_service)
+ }
+ }
+ linked_ids.uniq!
+ relations << { type: :same_route, linked_ids: linked_ids } if linked_ids.size > 1
+ else
+ # This shouldn't be possible
+ raise 'Unknown relation case in create_representative_vrp. If there is a new relation, update this function'
+ end
+ }
+
  # TODO: The following two "fake" vehicles can have carefully selected start and end points!
  # So that if there are multiple zone/cities or multiple depots, the split will be
  # more intelligent. For that we need go over the list of uniq depots and select two
@@ -521,7 +557,7 @@ def self.build_partial_service_vrp(service_vrp, partial_service_ids, available_v
  end
 
  # TODO: private method, reduce params
- def self.kmeans_process(nb_clusters, data_items, unit_symbols, limits, options = {}, &block)
+ def self.kmeans_process(nb_clusters, data_items, related_item_indices, limits, options = {}, &block)
  biggest_cluster_size = 0
  clusters = []
  restart = 0
@@ -544,7 +580,11 @@ def self.kmeans_process(nb_clusters, data_items, unit_symbols, limits, options =
  # TODO: move the creation of data_set to the gem side GEM should create it if necessary
  options[:seed] = rand(1234567890) # gem does not initialise the seed randomly
  log "BalancedVRPClustering is launched with seed #{options[:seed]}"
- c.build(DataSet.new(data_items: c.centroid_indices.empty? ? data_items : data_items.dup), options[:cut_symbol], ratio, options)
+ c.build(DataSet.new(data_items: Marshal.load(Marshal.dump(data_items))),
+ options[:cut_symbol],
+ Oj.load(Oj.dump(related_item_indices)),
+ ratio,
+ options)
 
  c.clusters.delete([])
  values = c.clusters.collect{ |cluster| cluster.data_items.collect{ |i| i[3][options[:cut_symbol]] }.sum.to_i }
@@ -607,7 +647,6 @@ def self.split_balanced_kmeans(service_vrp, nb_clusters, options = {}, &block)
  if vrp.shipments.all?{ |shipment| shipment&.pickup&.point&.location && shipment&.delivery&.point&.location } &&
  vrp.services.all?{ |service| service&.activity&.point&.location } && nb_clusters > 1
  cumulated_metrics = Hash.new(0)
- unit_symbols = vrp.units.collect{ |unit| unit.id.to_sym } << :duration << :visits
 
  if options[:entity] == :work_day || !vrp.matrices.empty?
  vrp.compute_matrix if vrp.matrices.empty?
@@ -627,7 +666,7 @@ def self.split_balanced_kmeans(service_vrp, nb_clusters, options = {}, &block)
 
  options[:clusters_infos] = collect_cluster_data(vrp, nb_clusters)
 
- clusters = kmeans_process(nb_clusters, data_items, unit_symbols, limits, options, &block)
+ clusters = kmeans_process(nb_clusters, data_items, related_item_indices, limits, options, &block)
 
  toc = Time.now
 

test/lib/interpreters/split_clustering_test.rb

@@ -565,7 +565,7 @@ def test_max_split_respects_initial_solutions
  problem[:vehicles] << problem[:vehicles].first.dup
  problem[:vehicles].last[:id] = 'vehicle_1'
  problem[:routes] = [
- # (1, 2) and (4, 5) are at the same locations without initial routes, they would be in the same vehicles
+ # (1, 2) and (4, 5) are at the same locations; without initial routes, they would end up in the same vehicles,
  # we are forcing them apart with initial routes and check if they stay as such after the split
  { vehicle_id: 'vehicle_0', mission_ids: [1, 4].map{ |s| "service_#{s}" } },
  { vehicle_id: 'vehicle_1', mission_ids: [2, 5].map{ |s| "service_#{s}" } }
@@ -589,9 +589,9 @@ def test_max_split_respects_initial_solutions
  def test_max_split_can_handle_empty_vehicles
  # Due to initial solutions, 4 services are on 2 vehicles which leaves 2 services to the remaining 3 vehicles.
  problem = VRP.lat_lon
- (1..4).each{ |i|
+ 4.times{ |i|
  problem[:vehicles] << problem[:vehicles].first.dup
- problem[:vehicles].last[:id] = "vehicle_#{i}"
+ problem[:vehicles].last[:id] = "vehicle_#{i + 1}"
  }
  problem[:routes] = [
  { vehicle_id: 'vehicle_0', mission_ids: [1, 4].map{ |s| "service_#{s}" } },
@@ -603,7 +603,7 @@ def test_max_split_can_handle_empty_vehicles
  Interpreters::SplitClustering.stub(:split_solve_core, lambda{ |service_vrp, _job|
  refute_nil service_vrp[:split_level], 'split_level should have been defined before split_solve_core'
  assert_operator service_vrp[:split_level], :<, 3,
- 'split_level should not reach 3. Grouping of vehicle points might be the reason'
+ 'Infinite loop?: split_level should not reach 3. Grouping of vehicle points might be the reason'
  assert service_vrp[:split_solve_data][:representative_vrp].points.none?{ |p| p.location.lat.nan? },
  'Empty vehicles should not reach split_solve_core'
  called = true
@@ -622,13 +622,22 @@ def test_max_split_can_handle_empty_vehicles
  assert called, 'split_solve_core should have been called'
  end
 
- def test_which_relations_are_linking
+ def test_which_relations_are_linking_and_forcing
  assert_equal %i[
  order
  same_route
  sequence
  shipment
  ], Interpreters::SplitClustering::LINKING_RELATIONS, 'Linking relation constant has changed'
+
+ assert_equal %i[
+ maximum_day_lapse
+ maximum_duration_lapse
+ meetup
+ minimum_day_lapse
+ minimum_duration_lapse
+ vehicle_trips
+ ], Interpreters::SplitClustering::FORCING_RELATIONS, 'Forcing relation constant has changed'
  end
 
  def test_collect_data_items_respects_linking_relations
@@ -659,6 +668,65 @@ def test_collect_data_items_respects_linking_relations
  assert_equal expected_linked_items, linked_items, 'Linking relations should link data_items together'
  end
 
+ def test_max_split_can_handle_pud_and_same_route_relations
+ # Services 1 and 2 are at the same location, as Services 4 and 5. And Services 3 and 6 are far away from tjh
+ # With Shipments s1{p:1, d:3}, s2{p:2, d:5}, s3{p:4, d:6}, we test that split does the "right" thing; even if,
+ # it is the "hardest" -- i.e., creating a "costly" split and forcing close points to be on different vehicles.
+ problem = VRP.lat_lon
+ 6.times{ |i| # have enough vehicles to see if the relations are respected
+ problem[:vehicles] << problem[:vehicles].first.dup
+ problem[:vehicles].last[:id] = "vehicle_#{i + 1}"
+ }
+
+ # Shipments are only supported as relations
+ problem[:relations] = [
+ { type: :shipment, linked_ids: [1, 3].collect{ |i| "service_#{i}" } },
+ { type: :shipment, linked_ids: [2, 5].collect{ |i| "service_#{i}" } },
+ { type: :same_route, linked_ids: [4, 6].collect{ |i| "service_#{i}" } },
+ { type: :meetup, linked_ids: %w[service_1 service_2] },
+ # { type: :same_route, linked_ids: [1, 2, 4].collect{ |i| "service_#{i}" } }
+ # { type: :vehicle_trips, linked_vehicle_ids: Array.new(7){ |i| "vehicle_#{i}" } },
+ ]
+ needs_to_stay_in_the_same_side = %w[service_1 service_2 service_3 service_5] # shipment + meet_up
+
+ # the split should stop even if there are "two services" (which is actually one shipment)
+ problem[:configuration][:preprocessing][:max_split_size] = 1
+
+ called = false
+ Interpreters::SplitClustering.stub(:split_solve_core, lambda{ |service_vrp, _job|
+ assert_operator service_vrp[:split_level], :<, 3, # infinite loop
+ 'Infinite loop?: split_level should not reach 3. Split should handle linking relations!'
+ called = true
+ Interpreters::SplitClustering.send(:__minitest_stub__split_solve_core, service_vrp) # call original function
+ }) do
+ OptimizerWrapper.stub(:solve, lambda{ |service_vrp, _job, _block| # stub with empty solution
+ vrp = service_vrp[:vrp]
+ # check that only necessary relations are present with all its services
+ assert_equal problem[:relations].count{ |r|
+ r[:linked_ids]&.any?{ |id| vrp.services.any?{ |s| s.id == id } } ||
+ r[:linked_vehicle_ids]&.any?{ |id| vrp.vehicles.any?{ |v| v.id == id } }
+ },
+ vrp.relations.size,
+ 'Split does not respect relations: missing/extra relations'
+
+ assert_empty vrp.relations.flat_map(&:linked_ids) - vrp.services.collect(&:id),
+ 'Split does not respect relations: missing/extra services'
+
+ assert needs_to_stay_in_the_same_side.all?{ |id| vrp.services.any?{ |s| s.id == id } } ||
+ needs_to_stay_in_the_same_side.none?{ |id| vrp.services.any?{ |s| s.id == id } },
+ "#{needs_to_stay_in_the_same_side} should stay on the same subproblem due to relations"
+
+ service = service_vrp[:service]
+ OptimizerWrapper.config[:services][service].detect_unfeasible_services(vrp)
+ OptimizerWrapper.config[:services][service].empty_result(service.to_s, vrp)
+ }) do
+ vrp = TestHelper.create(problem)
+ Interpreters::SplitClustering.split_solve({ service: :ortools, vrp: vrp, dicho_level: 0 })
+ end
+ end
+ assert called, 'split_solve_core should have been called'
+ end
+
  def test_ignore_debug_parameter_if_no_coordinates
  vrp = TestHelper.load_vrp(self)
  tmp_output_clusters = OptimizerWrapper.config[:debug][:output_clusters]