diff --git a/config/evo_quality_preset.ini b/config/evo_quality_preset.ini index 5f49770c6..0f7422f0f 100644 --- a/config/evo_quality_preset.ini +++ b/config/evo_quality_preset.ini @@ -109,7 +109,7 @@ max-steiner-tree-size=4 mapping-largest-he-fraction=0.0 mapping-min-pin-coverage=0.05 #main -> evolutionary -evolutionary.population_size=100; +evolutionary.population_size=10; evolutionary.dynamic_population_size=false; evolutionary.dynamic_population_amount_of_time=0.15; evolutionary.diversify_interval=1; diff --git a/config/evo_quality_preset_no_unconstrained.ini b/config/evo_quality_preset_no_unconstrained.ini new file mode 100644 index 000000000..becbb5737 --- /dev/null +++ b/config/evo_quality_preset_no_unconstrained.ini @@ -0,0 +1,117 @@ +# general +mode=direct +preset-type=quality +maxnet-removal-factor=0.01 +smallest-maxnet-threshold=50000 +maxnet-ignore=1000 +num-vcycles=0 +partition-evolutionary=true +time-limit=300 +# main -> shared_memory +s-use-localized-random-shuffle=false +s-static-balancing-work-packages=128 +# main -> preprocessing +p-enable-community-detection=true +# main -> preprocessing -> community_detection +p-louvain-edge-weight-function=hybrid +p-max-louvain-pass-iterations=5 +p-louvain-min-vertex-move-fraction=0.01 +p-vertex-degree-sampling-threshold=200000 +# main -> coarsening +c-type=multilevel_coarsener +c-use-adaptive-edge-size=true +c-min-shrink-factor=1.01 +c-max-shrink-factor=2.5 +c-s=1 +c-t=160 +c-vertex-degree-sampling-threshold=200000 +# main -> coarsening -> rating +c-rating-score=heavy_edge +c-rating-heavy-node-penalty=no_penalty +c-rating-acceptance-criterion=best_prefer_unmatched +# main -> initial_partitioning +i-mode=rb +i-runs=20 +i-use-adaptive-ip-runs=true +i-min-adaptive-ip-runs=5 +i-perform-refinement-on-best-partitions=true +i-fm-refinement-rounds=1 +i-lp-maximum-iterations=20 +i-lp-initial-block-size=5 +# main -> initial_partitioning -> refinement +i-r-refine-until-no-improvement=false +i-r-relative-improvement-threshold=0.0 +# main -> initial_partitioning -> refinement -> label_propagation +i-r-lp-type=label_propagation +i-r-lp-maximum-iterations=5 +i-r-lp-rebalancing=true +i-r-lp-he-size-activation-threshold=100 +# main -> initial_partitioning -> refinement -> fm +i-r-fm-type=kway_fm +i-r-fm-multitry-rounds=5 +i-r-fm-perform-moves-global=false +i-r-fm-rollback-parallel=true +i-r-fm-rollback-balance-violation-factor=1 +i-r-fm-seed-nodes=25 +i-r-fm-obey-minimal-parallelism=false +i-r-fm-release-nodes=true +i-r-fm-time-limit-factor=0.25 +i-r-fm-iter-moves-on-recalc=true +# main -> initial_partitioning -> refinement -> flows +i-r-flow-algo=do_nothing +# main -> refinement +r-rebalancer-type=advanced_rebalancer +r-refine-until-no-improvement=true +r-relative-improvement-threshold=0.0025 +# main -> refinement -> label_propagation +r-lp-type=do_nothing +r-lp-unconstrained=true +r-lp-maximum-iterations=5 +r-lp-rebalancing=true +r-lp-he-size-activation-threshold=100 +r-lp-relative-improvement-threshold=0.001 +# main -> refinement -> fm +r-fm-type=kway_fm +r-fm-multitry-rounds=10 +r-fm-unconstrained-rounds=8 +r-fm-perform-moves-global=false +r-fm-rollback-parallel=true +r-fm-rollback-balance-violation-factor=1.0 +r-fm-threshold-border-node-inclusion=0.7 +r-fm-imbalance-penalty-min=0.2 +r-fm-imbalance-penalty-max=1.0 +r-fm-seed-nodes=25 +r-fm-release-nodes=true +r-fm-min-improvement=-1.0 +r-fm-unconstrained-min-improvement=0.002 +r-fm-obey-minimal-parallelism=true +r-fm-time-limit-factor=0.25 +r-fm-iter-moves-on-recalc=true +# main -> refinement -> flows +r-flow-algo=flow_cutter +r-flow-scaling=16 +r-flow-max-num-pins=4294967295 +r-flow-find-most-balanced-cut=true +r-flow-determine-distance-from-cut=true +r-flow-parallel-search-multiplier=1.0 +r-flow-max-bfs-distance=2 +r-flow-min-relative-improvement-per-round=0.001 +r-flow-time-limit-factor=8 +r-flow-skip-small-cuts=true +r-flow-skip-unpromising-blocks=true +r-flow-pierce-in-bulk=true +r-flow-process-mapping-policy=lower_bound +# main -> mapping +one-to-one-mapping-strategy=greedy_mapping +mapping-use-local-search=true +use-two-phase-approach=false +max-steiner-tree-size=4 +mapping-largest-he-fraction=0.0 +mapping-min-pin-coverage=0.05 +#main -> evolutionary +evolutionary.population_size=10; +evolutionary.dynamic_population_size=false; +evolutionary.dynamic_population_amount_of_time=0.15; +evolutionary.diversify_interval=1; +evolutionary.mutation_chance=0.5; +evo-replacement-strategy=strong-diverse; diff --git a/config/evo_quality_preset_short.ini b/config/evo_quality_preset_short.ini new file mode 100644 index 000000000..da2443fd4 --- /dev/null +++ b/config/evo_quality_preset_short.ini @@ -0,0 +1,117 @@ +# general +mode=direct +preset-type=quality +maxnet-removal-factor=0.01 +smallest-maxnet-threshold=50000 +maxnet-ignore=1000 +num-vcycles=0 +partition-evolutionary=true +time-limit=20 +# main -> shared_memory +s-use-localized-random-shuffle=false +s-static-balancing-work-packages=128 +# main -> preprocessing +p-enable-community-detection=true +# main -> preprocessing -> community_detection +p-louvain-edge-weight-function=hybrid +p-max-louvain-pass-iterations=5 +p-louvain-min-vertex-move-fraction=0.01 +p-vertex-degree-sampling-threshold=200000 +# main -> coarsening +c-type=multilevel_coarsener +c-use-adaptive-edge-size=true +c-min-shrink-factor=1.01 +c-max-shrink-factor=2.5 +c-s=1 +c-t=160 +c-vertex-degree-sampling-threshold=200000 +# main -> coarsening -> rating +c-rating-score=heavy_edge +c-rating-heavy-node-penalty=no_penalty +c-rating-acceptance-criterion=best_prefer_unmatched +# main -> initial_partitioning +i-mode=rb +i-runs=20 +i-use-adaptive-ip-runs=true +i-min-adaptive-ip-runs=5 +i-perform-refinement-on-best-partitions=true +i-fm-refinement-rounds=1 +i-lp-maximum-iterations=20 +i-lp-initial-block-size=5 +# main -> initial_partitioning -> refinement +i-r-refine-until-no-improvement=false +i-r-relative-improvement-threshold=0.0 +# main -> initial_partitioning -> refinement -> label_propagation +i-r-lp-type=label_propagation +i-r-lp-maximum-iterations=5 +i-r-lp-rebalancing=true +i-r-lp-he-size-activation-threshold=100 +# main -> initial_partitioning -> refinement -> fm +i-r-fm-type=kway_fm +i-r-fm-multitry-rounds=5 +i-r-fm-perform-moves-global=false +i-r-fm-rollback-parallel=true +i-r-fm-rollback-balance-violation-factor=1 +i-r-fm-seed-nodes=25 +i-r-fm-obey-minimal-parallelism=false +i-r-fm-release-nodes=true +i-r-fm-time-limit-factor=0.25 +i-r-fm-iter-moves-on-recalc=true +# main -> initial_partitioning -> refinement -> flows +i-r-flow-algo=do_nothing +# main -> refinement +r-rebalancer-type=advanced_rebalancer +r-refine-until-no-improvement=true +r-relative-improvement-threshold=0.0025 +# main -> refinement -> label_propagation +r-lp-type=label_propagation +r-lp-unconstrained=true +r-lp-maximum-iterations=5 +r-lp-rebalancing=true +r-lp-he-size-activation-threshold=100 +r-lp-relative-improvement-threshold=0.001 +# main -> refinement -> fm +r-fm-type=unconstrained_fm +r-fm-multitry-rounds=10 +r-fm-unconstrained-rounds=8 +r-fm-perform-moves-global=false +r-fm-rollback-parallel=true +r-fm-rollback-balance-violation-factor=1.0 +r-fm-threshold-border-node-inclusion=0.7 +r-fm-imbalance-penalty-min=0.2 +r-fm-imbalance-penalty-max=1.0 +r-fm-seed-nodes=25 +r-fm-release-nodes=true +r-fm-min-improvement=-1.0 +r-fm-unconstrained-min-improvement=0.002 +r-fm-obey-minimal-parallelism=true +r-fm-time-limit-factor=0.25 +r-fm-iter-moves-on-recalc=true +# main -> refinement -> flows +r-flow-algo=flow_cutter +r-flow-scaling=16 +r-flow-max-num-pins=4294967295 +r-flow-find-most-balanced-cut=true +r-flow-determine-distance-from-cut=true +r-flow-parallel-search-multiplier=1.0 +r-flow-max-bfs-distance=2 +r-flow-min-relative-improvement-per-round=0.001 +r-flow-time-limit-factor=8 +r-flow-skip-small-cuts=true +r-flow-skip-unpromising-blocks=true +r-flow-pierce-in-bulk=true +r-flow-process-mapping-policy=lower_bound +# main -> mapping +one-to-one-mapping-strategy=greedy_mapping +mapping-use-local-search=true +use-two-phase-approach=false +max-steiner-tree-size=4 +mapping-largest-he-fraction=0.0 +mapping-min-pin-coverage=0.05 +#main -> evolutionary +evolutionary.population_size=10; +evolutionary.dynamic_population_size=false; +evolutionary.dynamic_population_amount_of_time=0.15; +evolutionary.diversify_interval=1; +evolutionary.mutation_chance=0.5; +evo-replacement-strategy=strong-diverse; diff --git a/config/evolutionary_preset.ini b/config/evolutionary_preset.ini index fd588720b..a46ed3a70 100644 --- a/config/evolutionary_preset.ini +++ b/config/evolutionary_preset.ini @@ -6,7 +6,7 @@ smallest-maxnet-threshold=50000 maxnet-ignore=1000 num-vcycles=0 partition-evolutionary=true -time-limit= 60 +time-limit= 20 #300 # main -> shared_memory s-use-localized-random-shuffle=false @@ -96,8 +96,7 @@ max-steiner-tree-size=4 mapping-largest-he-fraction=0.0 mapping-min-pin-coverage=0.05 #main -> evolutionary -evolutionary.population_size=1; -#100 +evolutionary.population_size=10; evolutionary.dynamic_population_size=false; evolutionary.dynamic_population_amount_of_time=0.15; evolutionary.diversify_interval=1; diff --git a/mt-kahypar/io/command_line_options.cpp b/mt-kahypar/io/command_line_options.cpp index 4ce19a952..1a8b0eb0d 100644 --- a/mt-kahypar/io/command_line_options.cpp +++ b/mt-kahypar/io/command_line_options.cpp @@ -738,7 +738,10 @@ namespace mt_kahypar { "default: 0.5)") ("evo-history-file", po::value(&context.evolutionary.history_file)->value_name(""), - "Output file for evolution history"); + "Output file for evolution history") + ("evo-kway", + po::value(&context.evolutionary.kway_combine)->value_name("")->default_value(2), + "How many individsuals to combine for the combine step\n"); return options; } diff --git a/mt-kahypar/partition/context.h b/mt-kahypar/partition/context.h index fe40587af..220dd9b25 100644 --- a/mt-kahypar/partition/context.h +++ b/mt-kahypar/partition/context.h @@ -281,6 +281,7 @@ struct EvolutionaryParameters { mutable int iteration; mutable std::chrono::milliseconds time_elapsed; std::string history_file; + int kway_combine; }; std::ostream & operator<< (std::ostream& str, const EvolutionaryParameters& params); diff --git a/mt-kahypar/partition/evo_partitioner.cpp b/mt-kahypar/partition/evo_partitioner.cpp index 9f1b4b171..7090dae8d 100644 --- a/mt-kahypar/partition/evo_partitioner.cpp +++ b/mt-kahypar/partition/evo_partitioner.cpp @@ -34,9 +34,15 @@ namespace mt_kahypar { timer.stop_timer("preprocessing"); Population population; - generateInitialPopulation(hypergraph, context, target_graph, population); + std::string history = generateInitialPopulation(hypergraph, context, target_graph, population); - performEvolution(hypergraph, context, target_graph, population); + history += performEvolution(hypergraph, context, target_graph, population); + + if (context.evolutionary.history_file != "") { + std::ofstream out_stream(context.evolutionary.history_file.c_str()); + out_stream << history; + out_stream.close(); + } PartitionedHypergraph partitioned_hypergraph(context.partition.k, hypergraph); @@ -90,21 +96,22 @@ namespace mt_kahypar { template - void EvoPartitioner::generateInitialPopulation(EvoPartitioner::Hypergraph& hg, Context& context, TargetGraph* target_graph, Population& population) { + std::string EvoPartitioner::generateInitialPopulation(EvoPartitioner::Hypergraph& hg, Context& context, TargetGraph* target_graph, Population& population) { context.partition.verbose_output = false; int timelimit = context.partition.time_limit; - //context.evolutionary.dynamic_population_size = true; - //context.evolutionary.population_size = 50; utils::Timer& timer = utils::Utilities::instance().getTimer(context.utility_id); auto start = std::chrono::duration_cast(std::chrono::high_resolution_clock::now().time_since_epoch()); auto now = std::chrono::duration_cast(std::chrono::high_resolution_clock::now().time_since_epoch()); auto time_elapsed = now - start; auto duration = std::chrono::seconds(timelimit); + std::string history = "Starttime: " + std::to_string(start.count()) + "\n"; // INITIAL POPULATION if (context.evolutionary.dynamic_population_size) { HighResClockTimepoint start = std::chrono::high_resolution_clock::now(); timer.start_timer("evolutionary", "Evolutionary"); - generateIndividual(hg, context, target_graph, population); + auto fitness = generateIndividual(hg, context, target_graph, population).fitness(); + now = std::chrono::duration_cast(std::chrono::high_resolution_clock::now().time_since_epoch()); + history += "" + std::to_string(now.count()) + ", Initial, " + std::to_string(fitness) + "\n"; timer.stop_timer("evolutionary"); ++context.evolutionary.iteration; @@ -117,18 +124,25 @@ namespace mt_kahypar { LOG << context.evolutionary.population_size; LOG << population; } + int best; + int iteration = 0; while (population.size() < context.evolutionary.population_size && time_elapsed <= duration) { - //timer.get("evolutionary") <= timelimit) { ++context.evolutionary.iteration; timer.start_timer("evolutionary", "Evolutionary"); - generateIndividual(hg, context, target_graph, population); + auto cur = generateIndividual(hg, context, target_graph, population).fitness(); timer.stop_timer("evolutionary"); now = std::chrono::duration_cast(std::chrono::high_resolution_clock::now().time_since_epoch()); + if (iteration == 0 || (cur < best)) { + best = cur; + history += "" + std::to_string(now.count()) + ", Repetition, " + std::to_string(cur) + "\n"; + } + iteration++; time_elapsed = now - start; } context.evolutionary.time_elapsed = time_elapsed; context.partition.verbose_output = true; + return history; } template @@ -200,53 +214,31 @@ namespace mt_kahypar { template std::string EvoPartitioner::performCombine(EvoPartitioner::Hypergraph& hypergraph, const Context& context, TargetGraph* target_graph, Population& population) { - //LOG << "Combining: "; - //const auto& parents = population.tournamentSelect(); - //std::vector part1 = parents.first.get().partition(); - //std::vector part2 = parents.second.get().partition() - const size_t position1 = population.randomIndividual(); - const size_t position2 = population.randomIndividualExcept(position1); - /*std::vector part1 = population.individualAt(position1).partition(); - std::vector part2 = population.individualAt(position2).partition(); - vec comms(hypergraph.initialNumNodes()); - for ( const HypernodeID& hn : hypergraph.nodes() ) { - comms[hn] = part1[hn] + context.partition.k * part2[hn]; - }*/ - std::vector parents = { - position1, - position2 - }; + std::vector parents; + size_t best = population.randomIndividual(); + parents.push_back(best); + for (int x = 1; x < context.evolutionary.kway_combine; x ++) { + size_t new_parent = population.randomIndividual(); + parents.push_back(new_parent); + if (population.individualAt(new_parent).fitness() <= population.individualAt(best).fitness()) { + best = new_parent; + } + } std::unordered_map comm_to_block; vec comms = combinePartitions(context, population, parents); vec part(hypergraph.initialNumNodes()); PartitionedHypergraph partitioned_hypergraph(context.partition.k, hypergraph); - size_t better = (population.individualAt(position1).fitness() <= population.individualAt(position2).fitness()) ? position1 : position2; for ( const HypernodeID& hn : hypergraph.nodes() ) { - //partitioned_hypergraph.setOnlyNodePart(hn, comms[hn]); - partitioned_hypergraph.setOnlyNodePart(hn, population.individualAt(better).partition()[hn]); + partitioned_hypergraph.setOnlyNodePart(hn, population.individualAt(best).partition()[hn]); if (comm_to_block.find(comms[hn]) == comm_to_block.end()) { - comm_to_block[comms[hn]] = population.individualAt(better).partition()[hn]; + comm_to_block[comms[hn]] = population.individualAt(best).partition()[hn]; } } partitioned_hypergraph.initializePartition(); hypergraph.setCommunityIDs(std::move(comms)); - /*vec comms = combinePartitions(context, population, parents); - vec part(hypergraph.initialNumNodes()); - - PartitionedHypergraph partitioned_hypergraph(context.partition.k, hypergraph); - size_t better = (population.individualAt(position1).fitness() <= population.individualAt(position2).fitness()) ? position1 : position2; - for ( const HypernodeID& hn : hypergraph.nodes() ) { - //partitioned_hypergraph.setOnlyNodePart(hn, comms[hn]); - partitioned_hypergraph.setOnlyNodePart(hn, population.individualAt(better).partition()[hn]); - part[hn] = population.individualAt(better).partition()[hn]; - } - partitioned_hypergraph.initializePartition(); - //hypergraph.setCommunityIDs(std::move(comms)); - hypergraph.setCommunityIDs(std::move(part));*/ if (context.partition.mode == Mode::direct) { Multilevel::evolutionPartitionVCycle(hypergraph, partitioned_hypergraph, context, comm_to_block, target_graph); - //partitioned_hypergraph = Multilevel::partition(hypergraph, context, target_graph); } else { throw InvalidParameterException("Invalid partitioning mode!"); } @@ -300,7 +292,7 @@ namespace mt_kahypar { throw InvalidParameterException("Invalid partitioning mode!"); } } - Individual individual(partitioned_hypergraph, context); + Individual individual(partitioned_hypergraph, context); //LOG << "Mutated Individual with fitness: " << population.individualAt(mutation_position).fitness() << " to " << individual.fitness(); std::string ret = ""; if (individual.fitness() < population.bestFitness()) { @@ -335,49 +327,15 @@ namespace mt_kahypar { } template - void EvoPartitioner::performEvolution(EvoPartitioner::Hypergraph& hg, Context& context, TargetGraph* target_graph, Population& population) { + std::string EvoPartitioner::performEvolution(EvoPartitioner::Hypergraph& hg, Context& context, TargetGraph* target_graph, Population& population) { context.partition.verbose_output = false; int timelimit = context.partition.time_limit; utils::Timer& timer = utils::Utilities::instance().getTimer(context.utility_id); int mutations = 0; int combinations = 0; auto time_start = std::chrono::duration_cast(std::chrono::high_resolution_clock::now().time_since_epoch()); - std::string history = "" + std::to_string(time_start.count()) + ", Initial, " + std::to_string(population.bestFitness()) + "\n"; + std::string history = ""; std::mutex _history_mutex; - /*while (timer.get("evolutionary") <= timelimit) { - ++context.evolutionary.iteration; - - timer.start_timer("evolutionary", "Evolutionary"); - - tbb::parallel_for(0, 4, [&](const int i) { - Context evo_context(context); - evo_context.type = ContextType::main; - evo_context.utility_id = utils::Utilities::instance().registerNewUtilityObjects(); - EvoDecision decision = decideNextMove(context); - EvoPartitioner::Hypergraph hg_copy = hg.copy(); - switch (decision) { - case EvoDecision::mutation: - { - std::lock_guard lock(_history_mutex); - history += performMutation(hg_copy, evo_context, target_graph, population); - mutations++; - break; - } - case EvoDecision::combine: - { - std::lock_guard lock(_history_mutex); - history += performCombine(hg_copy, evo_context, target_graph, population); - combinations++; - break; - } - default: - LOG << "Error in evo_partitioner.cpp: Non-covered case in decision making"; - std::exit(EXIT_FAILURE); - } - }); - timer.stop_timer("evolutionary"); - }*/ - auto duration = std::chrono::seconds(timelimit) - context.evolutionary.time_elapsed; std::atomic stop_flag(false); timer.start_timer("evolutionary", "Evolutionary"); @@ -427,11 +385,7 @@ namespace mt_kahypar { LOG << " " << mutations << " Mutations" << "\n"; LOG << " " << combinations << " Combinations" << "\n"; - if (context.evolutionary.history_file != "") { - std::ofstream out_stream(context.evolutionary.history_file.c_str()); - out_stream << history; - out_stream.close(); - } + return history; } } // namespace mt_kahypar \ No newline at end of file diff --git a/mt-kahypar/partition/evo_partitioner.h b/mt-kahypar/partition/evo_partitioner.h index 98892e2d5..94f36674e 100644 --- a/mt-kahypar/partition/evo_partitioner.h +++ b/mt-kahypar/partition/evo_partitioner.h @@ -18,7 +18,7 @@ class EvoPartitioner { static PartitionedHypergraph partition(Hypergraph& hypergraph, Context& context, TargetGraph* target_graph = nullptr); - static void generateInitialPopulation(EvoPartitioner::Hypergraph& hg, + static std::string generateInitialPopulation(EvoPartitioner::Hypergraph& hg, Context& context, TargetGraph* target_graph, Population& population); @@ -26,7 +26,7 @@ class EvoPartitioner { Context& context, TargetGraph* target_graph, Population& population); - static void performEvolution(EvoPartitioner::Hypergraph& hg, + static std::string performEvolution(EvoPartitioner::Hypergraph& hg, Context& context, TargetGraph* target_graph, Population& population);