From 8b04f56d63627fb6a9d840c6a0fc7d987b66aeb0 Mon Sep 17 00:00:00 2001 From: Zeke Morton Date: Wed, 12 Jun 2024 08:03:31 -0700 Subject: [PATCH 1/8] traverser: add support for or_slots Problem: the traverser does not support options for flexible scheduling. Add support for a logical or type of resource group, or_slots. or_slots are options for resource configurations that the traverser considers when selecting resources. --- resource/schema/data_std.cpp | 1 + resource/schema/data_std.hpp | 1 + resource/traversers/dfu_impl.cpp | 200 +++++++++++++++++++++++++++++-- resource/traversers/dfu_impl.hpp | 23 +++- 4 files changed, 216 insertions(+), 9 deletions(-) diff --git a/resource/schema/data_std.cpp b/resource/schema/data_std.cpp index 8407ac653..9d2f1a156 100644 --- a/resource/schema/data_std.cpp +++ b/resource/schema/data_std.cpp @@ -22,6 +22,7 @@ resource_type_t gpu_rt{"gpu"}; resource_type_t node_rt{"node"}; resource_type_t rack_rt{"rack"}; resource_type_t slot_rt{"slot"}; +resource_type_t or_slot_rt{"or_slot"}; } // namespace resource_model } // namespace Flux diff --git a/resource/schema/data_std.hpp b/resource/schema/data_std.hpp index 6bbca0fcf..b25371fe7 100644 --- a/resource/schema/data_std.hpp +++ b/resource/schema/data_std.hpp @@ -38,6 +38,7 @@ constexpr uint64_t resource_type_id{1}; struct resource_type_tag {}; using resource_type_t = intern::interned_string>; extern resource_type_t slot_rt; +extern resource_type_t or_slot_rt; extern resource_type_t cluster_rt; extern resource_type_t rack_rt; extern resource_type_t node_rt; diff --git a/resource/traversers/dfu_impl.cpp b/resource/traversers/dfu_impl.cpp index 0c6683932..5fbfd1a6f 100644 --- a/resource/traversers/dfu_impl.cpp +++ b/resource/traversers/dfu_impl.cpp @@ -259,33 +259,49 @@ int dfu_impl_t::match (vtx_t u, const std::vector &resources, const Resource **slot_resource, unsigned int *nslots, - const Resource **match_resource) + const Resource **match_resource, + const std::vector **slot_resources) { int rc = -1; - bool matched = false; + bool matched = false, or_matched = false; for (auto &resource : resources) { if ((*m_graph)[u].type == resource.type) { // Limitations of DFU traverser: jobspec must not // have same type at same level Please read utilities/README.md - if (matched == true) + if (matched || or_matched) goto ret; *match_resource = &resource; if (!resource.with.empty ()) { - for (auto &c_resource : resource.with) + for (auto &c_resource : resource.with) { + if (c_resource.type == or_slot_rt) { + *slot_resources = &resource.with; + *nslots = m_match->calc_effective_max (c_resource); + } if (c_resource.type == slot_rt) { *slot_resource = &c_resource; *nslots = m_match->calc_effective_max (c_resource); } + } } matched = true; } else if (resource.type == slot_rt) { // Limitations of DFU traverser: jobspec must not // have same type at same level Please read utilities/README.md - if (matched == true) + if (matched || or_matched) goto ret; *slot_resource = &resource; *nslots = m_match->calc_effective_max (resource); matched = true; + } else if (resource.type == or_slot_rt) { + // Limitations of DFU traverser: jobspec must not + // have same type at same level except for or_slot. + if (matched) + goto ret; + *slot_resources = &resources; + // This value is not well defined. In this state, nslots is + // determined by the last listed or_slot sibling in the jobspec. + *nslots = m_match->calc_effective_max (resource); + or_matched = true; } } rc = 0; @@ -340,14 +356,26 @@ const std::vector &dfu_impl_t::test (vtx_t u, const std::vector *ret = &resources; const Resource *slot_resources = NULL; const Resource *match_resources = NULL; - if (match (u, resources, &slot_resources, &nslots, &match_resources) < 0) { + const std::vector *slot_or_resources = NULL; + if (match (u, resources, &slot_resources, &nslots, &match_resources, &slot_or_resources) < 0) { m_err_msg += __FUNCTION__; m_err_msg += ": siblings in jobspec request same resource type "; m_err_msg += ": " + (*m_graph)[u].type + ".\n"; spec = match_kind_t::NONE_MATCH; goto done; } - if ((slot = slot_match (u, slot_resources))) { + if ((slot_or_resources)) { + // set default spec in case no match is found + spec = pristine ? match_kind_t::PRISTINE_NONE_MATCH : match_kind_t::NONE_MATCH; + + for (Resource r : *slot_or_resources) { + if ((slot_match (u, &r))) { + spec = match_kind_t::OR_SLOT_MATCH; + pristine = false; + ret = slot_or_resources; + } + } + } else if ((slot = slot_match (u, slot_resources))) { spec = match_kind_t::SLOT_MATCH; pristine = false; ret = &(slot_resources->with); @@ -707,6 +735,162 @@ int dfu_impl_t::dom_slot (const jobmeta_t &meta, return (qual_num_slots) ? 0 : -1; } +std::tuple dfu_impl_t::select_or_config ( + const std::vector &slots, + std::map resource_counts, + unsigned int nslots, + std::map> &or_config) +{ + int best = -1; + int i = -1; + std::string index = ""; + + // generate or_config index based on resource counts + for (auto it : resource_counts) + index = index + std::to_string (it.second) + " "; + + // if available, use precomputed result + auto it = or_config.find (index); + if (it != or_config.end ()) + return it->second; + + for (auto slot : slots) { + int test; + ++i; + bool match = true; + std::map updated_counts; + std::string updated_index = ""; + updated_counts = resource_counts; + + // determine if there are enough resources to match with this or_slot + for (auto slot_elem : slot.with) { + unsigned int qc = resource_counts[slot_elem.type]; + unsigned int count = m_match->calc_count (slot_elem, qc); + if (count <= 0) { + match = false; + break; + } + updated_counts[slot_elem.type] = updated_counts[slot_elem.type] - count; + } + if (!match) + continue; + + // find the best score after using resources from this or_slot + for (auto it : updated_counts) + updated_index = updated_index + std::to_string (it.second) + " "; + + test = std::get<1> (select_or_config (slots, updated_counts, nslots, or_config)); + if (best < test) { + best = test; + or_config[index] = std::make_tuple (updated_index, best + 1, i); + } + } + + // if there are no matches, set default score of 0 + // score represents the total number of or_slots that can be scheduled + // with optimal selection of or_slots + if (best < 0) { + or_config[index] = std::make_tuple ("", best + 1, -1); + } + return or_config[index]; +} +int dfu_impl_t::dom_or_slot (const jobmeta_t &meta, + vtx_t u, + const std::vector &slots, + unsigned int nslots, + bool pristine, + bool *excl, + scoring_api_t &dfu) +{ + int rc; + bool x_inout = true; + unsigned int qual_num_slots = 0; + std::vector edg_group_vector; + const subsystem_t &dom = m_match->dom_subsystem (); + std::unordered_set edges_used; + scoring_api_t dfu_slot; + std::map> or_config; + std::tuple current_config; + + // collect a set of all resource types in the or_slots to get resource + // counts. This does not work well with non leaf vertex resources because + // it cannot distinguish beyond type. This may be resolveable if graph + // coloring is removed during the selection process. + std::vector slot_resource_union; + std::map resource_types; + for (auto &slot : slots) { + for (auto r : slot.with) { + if (resource_types.find (r.type) == resource_types.end ()) { + resource_types[r.type] = 0; + slot_resource_union.push_back (r); + } + } + } + + if ((rc = explore (meta, + u, + dom, + slot_resource_union, + pristine, + &x_inout, + visit_t::DFV, + dfu_slot, + nslots)) + != 0) + goto done; + if ((rc = m_match->dom_finish_slot (dom, dfu_slot)) != 0) + goto done; + + for (auto &it : resource_types) { + it.second = dfu_slot.qualified_count (dom, it.first); + } + + // calculate the ideal or_slot config for avail resources. + // tuple is (key to next best option, current score, index of current best or_slot) + current_config = select_or_config (slots, resource_types, nslots, or_config); + + qual_num_slots = std::get<1> (current_config); + for (unsigned int i = 0; i < qual_num_slots; ++i) { + auto slot_index = std::get<2> (current_config); + eval_egroup_t edg_group; + int64_t score = MATCH_MET; + + // use calculated index to determine which or_slot type to use + for (auto &slot_elem : slots[slot_index].with) { + unsigned int j = 0; + unsigned int qc = dfu_slot.qualified_count (dom, slot_elem.type); + unsigned int count = m_match->calc_count (slot_elem, qc); + while (j < count) { + auto egroup_i = dfu_slot.eval_egroups_iter_next (dom, slot_elem.type); + if (egroup_i == dfu_slot.eval_egroups_end (dom, slot_elem.type)) { + m_err_msg += __FUNCTION__; + m_err_msg += ": not enough slots.\n"; + qual_num_slots = 0; + goto done; + } + eval_edg_t ev_edg ((*egroup_i).edges[0].count, + (*egroup_i).edges[0].count, + 1, + (*egroup_i).edges[0].edge); + score += (*egroup_i).score; + edg_group.edges.push_back (ev_edg); + j += (*egroup_i).edges[0].count; + } + } + edg_group.score = score; + edg_group.count = 1; + edg_group.exclusive = 1; + edg_group_vector.push_back (edg_group); + + current_config = or_config[std::get<0> (current_config)]; + } + for (auto &edg_group : edg_group_vector) + dfu.add (dom, or_slot_rt, edg_group); + +done: + return (qual_num_slots) ? 0 : -1; +} + int dfu_impl_t::dom_dfv (const jobmeta_t &meta, vtx_t u, const std::vector &resources, @@ -736,6 +920,8 @@ int dfu_impl_t::dom_dfv (const jobmeta_t &meta, (*m_graph)[u].idata.colors[dom] = m_color.gray (); if (sm == match_kind_t::SLOT_MATCH) dom_slot (meta, u, next, nslots, check_pres, &x_inout, dfu); + else if (sm == match_kind_t::OR_SLOT_MATCH) + dom_or_slot (meta, u, next, nslots, check_pres, &x_inout, dfu); else dom_exp (meta, u, next, check_pres, &x_inout, dfu); *excl = x_in; diff --git a/resource/traversers/dfu_impl.hpp b/resource/traversers/dfu_impl.hpp index ad78c11c9..1cecd59e7 100644 --- a/resource/traversers/dfu_impl.hpp +++ b/resource/traversers/dfu_impl.hpp @@ -32,7 +32,13 @@ namespace detail { enum class visit_t { DFV, UPV }; -enum class match_kind_t { RESOURCE_MATCH, SLOT_MATCH, NONE_MATCH, PRISTINE_NONE_MATCH }; +enum class match_kind_t { + RESOURCE_MATCH, + SLOT_MATCH, + OR_SLOT_MATCH, + NONE_MATCH, + PRISTINE_NONE_MATCH +}; struct jobmeta_t { enum class alloc_type_t : int { @@ -372,7 +378,8 @@ class dfu_impl_t { const std::vector &resources, const Jobspec::Resource **slot_resource, unsigned int *nslots, - const Jobspec::Resource **match_resource); + const Jobspec::Resource **match_resource, + const std::vector **slot_resources); bool slot_match (vtx_t u, const Jobspec::Resource *slot_resource); const std::vector &test (vtx_t u, const std::vector &resources, @@ -450,6 +457,18 @@ class dfu_impl_t { bool prestine, bool *excl, scoring_api_t &dfu); + std::tuple select_or_config ( + const std::vector &slots, + std::map resource_counts, + unsigned int nslots, + std::map> &or_config); + int dom_or_slot (const jobmeta_t &meta, + vtx_t u, + const std::vector &resources, + unsigned int nslots, + bool prestine, + bool *excl, + scoring_api_t &dfu); int dom_exp (const jobmeta_t &meta, vtx_t u, const std::vector &resources, From 59bd1300a1c3c8a9eedc22d14013ba1b8b09a5a3 Mon Sep 17 00:00:00 2001 From: Zeke Morton Date: Mon, 16 Sep 2024 13:42:53 -0700 Subject: [PATCH 2/8] traverser: add min accum for prime jobspec Problem: The traverser primes the jobspec with count of resources that are specified as pruning filter types. This additive accumulation results in counts that could be much higher than available counts in the planner when using flexible scheduling with or_slots. These high counts cause the pruning by subplanner to stop the traversal. This results in matches not being found when matches are available. Add in a new accumulation option min_if. This takes the lowest count instead of the sum of all resource counts. Use this when the parent type is or_slot_rt. --- resource/traversers/dfu_impl.cpp | 36 ++++++++++++++++++++++++++++++-- resource/traversers/dfu_impl.hpp | 10 +++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/resource/traversers/dfu_impl.cpp b/resource/traversers/dfu_impl.cpp index 5fbfd1a6f..b03e4a1c2 100644 --- a/resource/traversers/dfu_impl.cpp +++ b/resource/traversers/dfu_impl.cpp @@ -425,6 +425,23 @@ int dfu_impl_t::accum_if (subsystem_t subsystem, return rc; } +/* Same as above except that lowest is unorder_map */ +int dfu_impl_t::min_if (subsystem_t subsystem, + resource_type_t type, + unsigned int counts, + std::unordered_map &lowest) +{ + int rc = -1; + if (m_match->is_pruning_type (subsystem, type)) { + if (lowest.find (type) == lowest.end ()) + lowest[type] = counts; + else if (lowest[type] > counts) + lowest[type] = counts; + rc = 0; + } + return rc; +} + int dfu_impl_t::prime_exp (subsystem_t subsystem, vtx_t u, std::map &dfv) { int rc = 0; @@ -1340,8 +1357,23 @@ void dfu_impl_t::prime_jobspec (std::vector &resources, // as far as a subtree satisfies the minimum requirement accum_if (subsystem, resource.type, resource.count.min, to_parent); prime_jobspec (resource.with, resource.user_data); - for (auto &aggregate : resource.user_data) { - accum_if (subsystem, aggregate.first, resource.count.min * aggregate.second, to_parent); + + // Or slots should use a minimum of values rather than an accumulation + // otherwise possible matches may be filtered out + if (resource.type == or_slot_rt) { + for (auto &aggregate : resource.user_data) { + min_if (subsystem, + aggregate.first, + resource.count.min * aggregate.second, + to_parent); + } + } else { + for (auto &aggregate : resource.user_data) { + accum_if (subsystem, + aggregate.first, + resource.count.min * aggregate.second, + to_parent); + } } } } diff --git a/resource/traversers/dfu_impl.hpp b/resource/traversers/dfu_impl.hpp index 1cecd59e7..4ff06a321 100644 --- a/resource/traversers/dfu_impl.hpp +++ b/resource/traversers/dfu_impl.hpp @@ -402,6 +402,16 @@ class dfu_impl_t { unsigned int count, std::unordered_map &accum); + /*! Find min count if type matches with one of the resource + * types used in the scheduler-driven aggregate update (SDAU) scheme. + * dfu_match_cb_t provides an interface to configure what types are used + * for SDAU scheme. + */ + int min_if (subsystem_t subsystem, + resource_type_t type, + unsigned int count, + std::unordered_map &lowest); + // Explore out-edges for priming the subtree plans int prime_exp (subsystem_t subsystem, vtx_t u, std::map &dfv); From 05e5e0ce7a138bac4ae9bdcea7307a9a42a68706 Mon Sep 17 00:00:00 2001 From: Zeke Morton Date: Mon, 16 Sep 2024 14:14:58 -0700 Subject: [PATCH 3/8] tests: add or_slot flexible tests Problem: there are no tests for or_slots Add tests --- t/CMakeLists.txt | 1 + t/data/resource/commands/flexible/cmds01.in | 10 ++ t/data/resource/commands/flexible/cmds02.in | 6 + t/data/resource/commands/flexible/cmds03.in | 6 + t/data/resource/commands/flexible/cmds04.in | 2 + t/data/resource/commands/flexible/cmds05.in | 3 + t/data/resource/commands/flexible/cmds06.in | 2 + t/data/resource/commands/flexible/cmds07.in | 2 + t/data/resource/commands/flexible/cmds08.in | 2 + t/data/resource/expected/flexible/001.R.out | 152 ++++++++++++++++++ t/data/resource/expected/flexible/002.R.out | 76 +++++++++ t/data/resource/expected/flexible/003.R.out | 136 ++++++++++++++++ t/data/resource/expected/flexible/004.R.out | 4 + t/data/resource/expected/flexible/005.R.out | 53 ++++++ t/data/resource/expected/flexible/006.R.out | 0 t/data/resource/expected/flexible/007.R.out | 4 + t/data/resource/expected/flexible/008.R.out | 10 ++ .../resource/jobspecs/flexible/test001.yaml | 38 +++++ .../resource/jobspecs/flexible/test002.yaml | 37 +++++ .../resource/jobspecs/flexible/test003.yaml | 40 +++++ .../resource/jobspecs/flexible/test004.yaml | 28 ++++ .../resource/jobspecs/flexible/test005.yaml | 44 +++++ .../resource/jobspecs/flexible/test006.yaml | 21 +++ .../resource/jobspecs/flexible/test007.yaml | 27 ++++ .../resource/jobspecs/flexible/test008.yaml | 25 +++ t/t3037-resource-flexible.t | 76 +++++++++ 26 files changed, 805 insertions(+) create mode 100644 t/data/resource/commands/flexible/cmds01.in create mode 100644 t/data/resource/commands/flexible/cmds02.in create mode 100644 t/data/resource/commands/flexible/cmds03.in create mode 100644 t/data/resource/commands/flexible/cmds04.in create mode 100644 t/data/resource/commands/flexible/cmds05.in create mode 100644 t/data/resource/commands/flexible/cmds06.in create mode 100644 t/data/resource/commands/flexible/cmds07.in create mode 100644 t/data/resource/commands/flexible/cmds08.in create mode 100644 t/data/resource/expected/flexible/001.R.out create mode 100644 t/data/resource/expected/flexible/002.R.out create mode 100644 t/data/resource/expected/flexible/003.R.out create mode 100644 t/data/resource/expected/flexible/004.R.out create mode 100644 t/data/resource/expected/flexible/005.R.out create mode 100644 t/data/resource/expected/flexible/006.R.out create mode 100644 t/data/resource/expected/flexible/007.R.out create mode 100644 t/data/resource/expected/flexible/008.R.out create mode 100644 t/data/resource/jobspecs/flexible/test001.yaml create mode 100644 t/data/resource/jobspecs/flexible/test002.yaml create mode 100644 t/data/resource/jobspecs/flexible/test003.yaml create mode 100644 t/data/resource/jobspecs/flexible/test004.yaml create mode 100644 t/data/resource/jobspecs/flexible/test005.yaml create mode 100644 t/data/resource/jobspecs/flexible/test006.yaml create mode 100644 t/data/resource/jobspecs/flexible/test007.yaml create mode 100644 t/data/resource/jobspecs/flexible/test008.yaml create mode 100755 t/t3037-resource-flexible.t diff --git a/t/CMakeLists.txt b/t/CMakeLists.txt index 82368ac13..296818ea1 100644 --- a/t/CMakeLists.txt +++ b/t/CMakeLists.txt @@ -67,6 +67,7 @@ set(ALL_TESTS t3034-resource-pconstraints.t t3035-resource-remove.t t3036-rq2.t + t3037-resource-flexible.t t3300-system-dontblock.t t3301-system-latestart.t t4000-match-params.t diff --git a/t/data/resource/commands/flexible/cmds01.in b/t/data/resource/commands/flexible/cmds01.in new file mode 100644 index 000000000..e850b0f21 --- /dev/null +++ b/t/data/resource/commands/flexible/cmds01.in @@ -0,0 +1,10 @@ +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test001.yaml +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test001.yaml +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test001.yaml +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test001.yaml +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test001.yaml +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test001.yaml +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test001.yaml +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test001.yaml +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test001.yaml +quit diff --git a/t/data/resource/commands/flexible/cmds02.in b/t/data/resource/commands/flexible/cmds02.in new file mode 100644 index 000000000..eaad716c3 --- /dev/null +++ b/t/data/resource/commands/flexible/cmds02.in @@ -0,0 +1,6 @@ +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test002.yaml +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test002.yaml +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test002.yaml +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test002.yaml +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test002.yaml +quit diff --git a/t/data/resource/commands/flexible/cmds03.in b/t/data/resource/commands/flexible/cmds03.in new file mode 100644 index 000000000..59a7e82db --- /dev/null +++ b/t/data/resource/commands/flexible/cmds03.in @@ -0,0 +1,6 @@ +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test003.yaml +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test003.yaml +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test003.yaml +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test003.yaml +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test003.yaml +quit diff --git a/t/data/resource/commands/flexible/cmds04.in b/t/data/resource/commands/flexible/cmds04.in new file mode 100644 index 000000000..ca8695326 --- /dev/null +++ b/t/data/resource/commands/flexible/cmds04.in @@ -0,0 +1,2 @@ +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test004.yaml +quit diff --git a/t/data/resource/commands/flexible/cmds05.in b/t/data/resource/commands/flexible/cmds05.in new file mode 100644 index 000000000..f71a0e3fc --- /dev/null +++ b/t/data/resource/commands/flexible/cmds05.in @@ -0,0 +1,3 @@ +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test005.yaml +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test005.yaml +quit diff --git a/t/data/resource/commands/flexible/cmds06.in b/t/data/resource/commands/flexible/cmds06.in new file mode 100644 index 000000000..b259537a5 --- /dev/null +++ b/t/data/resource/commands/flexible/cmds06.in @@ -0,0 +1,2 @@ +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test006.yaml +quit diff --git a/t/data/resource/commands/flexible/cmds07.in b/t/data/resource/commands/flexible/cmds07.in new file mode 100644 index 000000000..1c9652d87 --- /dev/null +++ b/t/data/resource/commands/flexible/cmds07.in @@ -0,0 +1,2 @@ +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test007.yaml +quit diff --git a/t/data/resource/commands/flexible/cmds08.in b/t/data/resource/commands/flexible/cmds08.in new file mode 100644 index 000000000..05f1daca2 --- /dev/null +++ b/t/data/resource/commands/flexible/cmds08.in @@ -0,0 +1,2 @@ +match allocate @TEST_SRCDIR@/data/resource/jobspecs/flexible/test008.yaml +quit diff --git a/t/data/resource/expected/flexible/001.R.out b/t/data/resource/expected/flexible/001.R.out new file mode 100644 index 000000000..3de8f0a0e --- /dev/null +++ b/t/data/resource/expected/flexible/001.R.out @@ -0,0 +1,152 @@ + ---------------core28[1:x] + ---------------core29[1:x] + ---------------core30[1:x] + ---------------core31[1:x] + ---------------core32[1:x] + ---------------core33[1:x] + ---------------core34[1:x] + ---------------core35[1:x] + ---------------gpu1[1:x] + ------------socket1[1:s] + ---------node1[1:s] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=1 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= + ---------------core18[1:x] + ---------------core19[1:x] + ---------------core20[1:x] + ---------------core21[1:x] + ---------------core22[1:x] + ---------------core23[1:x] + ---------------core24[1:x] + ---------------core25[1:x] + ---------------core26[1:x] + ---------------core27[1:x] + ------------socket1[1:s] + ---------node1[1:s] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=2 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= + ---------------core10[1:x] + ---------------core11[1:x] + ---------------core12[1:x] + ---------------core13[1:x] + ---------------core14[1:x] + ---------------core15[1:x] + ---------------core16[1:x] + ---------------core17[1:x] + ---------------gpu0[1:x] + ------------socket0[1:s] + ---------node1[1:s] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=3 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= + ---------------core0[1:x] + ---------------core1[1:x] + ---------------core2[1:x] + ---------------core3[1:x] + ---------------core4[1:x] + ---------------core5[1:x] + ---------------core6[1:x] + ---------------core7[1:x] + ---------------core8[1:x] + ---------------core9[1:x] + ------------socket0[1:s] + ---------node1[1:s] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=4 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= + ---------------core28[1:x] + ---------------core29[1:x] + ---------------core30[1:x] + ---------------core31[1:x] + ---------------core32[1:x] + ---------------core33[1:x] + ---------------core34[1:x] + ---------------core35[1:x] + ---------------gpu1[1:x] + ------------socket1[1:s] + ---------node0[1:s] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=5 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= + ---------------core18[1:x] + ---------------core19[1:x] + ---------------core20[1:x] + ---------------core21[1:x] + ---------------core22[1:x] + ---------------core23[1:x] + ---------------core24[1:x] + ---------------core25[1:x] + ---------------core26[1:x] + ---------------core27[1:x] + ------------socket1[1:s] + ---------node0[1:s] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=6 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= + ---------------core10[1:x] + ---------------core11[1:x] + ---------------core12[1:x] + ---------------core13[1:x] + ---------------core14[1:x] + ---------------core15[1:x] + ---------------core16[1:x] + ---------------core17[1:x] + ---------------gpu0[1:x] + ------------socket0[1:s] + ---------node0[1:s] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=7 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= + ---------------core0[1:x] + ---------------core1[1:x] + ---------------core2[1:x] + ---------------core3[1:x] + ---------------core4[1:x] + ---------------core5[1:x] + ---------------core6[1:x] + ---------------core7[1:x] + ---------------core8[1:x] + ---------------core9[1:x] + ------------socket0[1:s] + ---------node0[1:s] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=8 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= +INFO: ============================= +INFO: No matching resources found +INFO: JOBID=9 +INFO: ============================= diff --git a/t/data/resource/expected/flexible/002.R.out b/t/data/resource/expected/flexible/002.R.out new file mode 100644 index 000000000..c3a5b5570 --- /dev/null +++ b/t/data/resource/expected/flexible/002.R.out @@ -0,0 +1,76 @@ + ---------------core28[1:x] + ---------------core29[1:x] + ---------------core30[1:x] + ---------------core31[1:x] + ---------------core32[1:x] + ---------------core33[1:x] + ---------------core34[1:x] + ---------------core35[1:x] + ---------------gpu1[1:x] + ------------socket1[1:s] + ---------node1[1:s] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=1 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= + ---------------core10[1:x] + ---------------core11[1:x] + ---------------core12[1:x] + ---------------core13[1:x] + ---------------core14[1:x] + ---------------core15[1:x] + ---------------core16[1:x] + ---------------core17[1:x] + ---------------gpu0[1:x] + ------------socket0[1:s] + ---------node1[1:s] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=2 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= + ---------------core28[1:x] + ---------------core29[1:x] + ---------------core30[1:x] + ---------------core31[1:x] + ---------------core32[1:x] + ---------------core33[1:x] + ---------------core34[1:x] + ---------------core35[1:x] + ---------------gpu1[1:x] + ------------socket1[1:s] + ---------node0[1:s] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=3 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= + ---------------core10[1:x] + ---------------core11[1:x] + ---------------core12[1:x] + ---------------core13[1:x] + ---------------core14[1:x] + ---------------core15[1:x] + ---------------core16[1:x] + ---------------core17[1:x] + ---------------gpu0[1:x] + ------------socket0[1:s] + ---------node0[1:s] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=4 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= +INFO: ============================= +INFO: No matching resources found +INFO: JOBID=5 +INFO: ============================= diff --git a/t/data/resource/expected/flexible/003.R.out b/t/data/resource/expected/flexible/003.R.out new file mode 100644 index 000000000..b7380276c --- /dev/null +++ b/t/data/resource/expected/flexible/003.R.out @@ -0,0 +1,136 @@ + ---------------core24[1:x] + ---------------core25[1:x] + ---------------core26[1:x] + ---------------core27[1:x] + ---------------core28[1:x] + ---------------core29[1:x] + ---------------core30[1:x] + ---------------core31[1:x] + ---------------core32[1:x] + ---------------core33[1:x] + ---------------core34[1:x] + ---------------core35[1:x] + ---------------memory6[2:x] + ---------------memory7[2:x] + ------------socket1[1:s] + ---------node0[1:s] + ---------------core24[1:x] + ---------------core25[1:x] + ---------------core26[1:x] + ---------------core27[1:x] + ---------------core28[1:x] + ---------------core29[1:x] + ---------------core30[1:x] + ---------------core31[1:x] + ---------------core32[1:x] + ---------------core33[1:x] + ---------------core34[1:x] + ---------------core35[1:x] + ---------------memory6[2:x] + ---------------memory7[2:x] + ------------socket1[1:s] + ---------node1[1:s] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=1 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= + ---------------core6[1:x] + ---------------core7[1:x] + ---------------core8[1:x] + ---------------core9[1:x] + ---------------core10[1:x] + ---------------core11[1:x] + ---------------core12[1:x] + ---------------core13[1:x] + ---------------core14[1:x] + ---------------core15[1:x] + ---------------core16[1:x] + ---------------core17[1:x] + ---------------memory2[2:x] + ---------------memory3[2:x] + ------------socket0[1:s] + ---------node0[1:s] + ---------------core6[1:x] + ---------------core7[1:x] + ---------------core8[1:x] + ---------------core9[1:x] + ---------------core10[1:x] + ---------------core11[1:x] + ---------------core12[1:x] + ---------------core13[1:x] + ---------------core14[1:x] + ---------------core15[1:x] + ---------------core16[1:x] + ---------------core17[1:x] + ---------------memory2[2:x] + ---------------memory3[2:x] + ------------socket0[1:s] + ---------node1[1:s] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=2 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= + ---------------core19[1:x] + ---------------core20[1:x] + ---------------core21[1:x] + ---------------core22[1:x] + ---------------core23[1:x] + ---------------gpu1[1:x] + ---------------memory4[2:x] + ---------------memory5[2:x] + ------------socket1[1:s] + ---------node0[1:s] + ---------------core19[1:x] + ---------------core20[1:x] + ---------------core21[1:x] + ---------------core22[1:x] + ---------------core23[1:x] + ---------------gpu1[1:x] + ---------------memory4[2:x] + ---------------memory5[2:x] + ------------socket1[1:s] + ---------node1[1:s] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=3 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= + ---------------core1[1:x] + ---------------core2[1:x] + ---------------core3[1:x] + ---------------core4[1:x] + ---------------core5[1:x] + ---------------gpu0[1:x] + ---------------memory0[2:x] + ---------------memory1[2:x] + ------------socket0[1:s] + ---------node0[1:s] + ---------------core1[1:x] + ---------------core2[1:x] + ---------------core3[1:x] + ---------------core4[1:x] + ---------------core5[1:x] + ---------------gpu0[1:x] + ---------------memory0[2:x] + ---------------memory1[2:x] + ------------socket0[1:s] + ---------node1[1:s] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=4 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= +INFO: ============================= +INFO: No matching resources found +INFO: JOBID=5 +INFO: ============================= diff --git a/t/data/resource/expected/flexible/004.R.out b/t/data/resource/expected/flexible/004.R.out new file mode 100644 index 000000000..5e4d95e5b --- /dev/null +++ b/t/data/resource/expected/flexible/004.R.out @@ -0,0 +1,4 @@ +INFO: ============================= +INFO: No matching resources found +INFO: JOBID=1 +INFO: ============================= diff --git a/t/data/resource/expected/flexible/005.R.out b/t/data/resource/expected/flexible/005.R.out new file mode 100644 index 000000000..a2db7966f --- /dev/null +++ b/t/data/resource/expected/flexible/005.R.out @@ -0,0 +1,53 @@ + ---------------core13[1:x] + ---------------core14[1:x] + ---------------core15[1:x] + ---------------core16[1:x] + ---------------core17[1:x] + ---------------gpu0[1:x] + ---------------memory1[2:x] + ---------------memory2[2:x] + ---------------memory3[2:x] + ------------socket0[1:x] + ---------------core31[1:x] + ---------------core32[1:x] + ---------------core33[1:x] + ---------------core34[1:x] + ---------------core35[1:x] + ---------------gpu1[1:x] + ---------------memory5[2:x] + ---------------memory6[2:x] + ---------------memory7[2:x] + ------------socket1[1:x] + ---------node0[1:x] + ---------------core13[1:x] + ---------------core14[1:x] + ---------------core15[1:x] + ---------------core16[1:x] + ---------------core17[1:x] + ---------------gpu0[1:x] + ---------------memory1[2:x] + ---------------memory2[2:x] + ---------------memory3[2:x] + ------------socket0[1:x] + ---------------core31[1:x] + ---------------core32[1:x] + ---------------core33[1:x] + ---------------core34[1:x] + ---------------core35[1:x] + ---------------gpu1[1:x] + ---------------memory5[2:x] + ---------------memory6[2:x] + ---------------memory7[2:x] + ------------socket1[1:x] + ---------node1[1:x] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=1 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= +INFO: ============================= +INFO: No matching resources found +INFO: JOBID=2 +INFO: ============================= diff --git a/t/data/resource/expected/flexible/006.R.out b/t/data/resource/expected/flexible/006.R.out new file mode 100644 index 000000000..e69de29bb diff --git a/t/data/resource/expected/flexible/007.R.out b/t/data/resource/expected/flexible/007.R.out new file mode 100644 index 000000000..5e4d95e5b --- /dev/null +++ b/t/data/resource/expected/flexible/007.R.out @@ -0,0 +1,4 @@ +INFO: ============================= +INFO: No matching resources found +INFO: JOBID=1 +INFO: ============================= diff --git a/t/data/resource/expected/flexible/008.R.out b/t/data/resource/expected/flexible/008.R.out new file mode 100644 index 000000000..7aefd9083 --- /dev/null +++ b/t/data/resource/expected/flexible/008.R.out @@ -0,0 +1,10 @@ + ---------------core35[1:x] + ------------socket1[1:s] + ---------node1[1:s] + ------rack0[1:s] + ---tiny0[1:s] +INFO: ============================= +INFO: JOBID=1 +INFO: RESOURCES=ALLOCATED +INFO: SCHEDULED AT=Now +INFO: ============================= diff --git a/t/data/resource/jobspecs/flexible/test001.yaml b/t/data/resource/jobspecs/flexible/test001.yaml new file mode 100644 index 000000000..cbfaf5bcb --- /dev/null +++ b/t/data/resource/jobspecs/flexible/test001.yaml @@ -0,0 +1,38 @@ +version: 9999 +resources: + - type: cluster + count: 1 + with: + - type: rack + count: 1 + with: + - type: node + count: 1 + with: + - type: socket + count: 1 + with: + - type: or_slot + count: 1 + label: small + with: + - type: core + count: 8 + - type: gpu + count: 1 + - type: or_slot + count: 1 + label: big + with: + - type: core + count: 10 +# a comment +attributes: + system: + duration: 3600 +tasks: + - command: [ "app" ] + slot: default + count: + per_slot: 1 + diff --git a/t/data/resource/jobspecs/flexible/test002.yaml b/t/data/resource/jobspecs/flexible/test002.yaml new file mode 100644 index 000000000..2f627b5f9 --- /dev/null +++ b/t/data/resource/jobspecs/flexible/test002.yaml @@ -0,0 +1,37 @@ +version: 9999 +resources: + - type: cluster + count: 1 + with: + - type: rack + count: 1 + with: + - type: node + count: 1 + with: + - type: socket + count: 1 + with: + - type: or_slot + count: 1 + label: small + with: + - type: core + count: 8 + - type: gpu + count: 1 + - type: or_slot + count: 1 + label: big + with: + - type: core + count: 36 +# a comment +attributes: + system: + duration: 3600 +tasks: + - command: [ "app" ] + slot: default + count: + per_slot: 1 \ No newline at end of file diff --git a/t/data/resource/jobspecs/flexible/test003.yaml b/t/data/resource/jobspecs/flexible/test003.yaml new file mode 100644 index 000000000..604e9967b --- /dev/null +++ b/t/data/resource/jobspecs/flexible/test003.yaml @@ -0,0 +1,40 @@ +version: 9999 +resources: + - type: or_slot + count: 2 + label: default + with: + - type: core + count: 12 + - type: memory + count: 4 + - type: or_slot + count: 2 + label: default + with: + - type: core + count: 5 + - type: gpu + count: 1 + - type: memory + count: 4 + - type: or_slot + count: 2 + label: default + with: + - type: core + count: 4 + - type: gpu + count: 2 + - type: memory + count: 4 + +# a comment +attributes: + system: + duration: 3600 +tasks: + - command: [ "app" ] + slot: default + count: + per_slot: 1 diff --git a/t/data/resource/jobspecs/flexible/test004.yaml b/t/data/resource/jobspecs/flexible/test004.yaml new file mode 100644 index 000000000..b921f3115 --- /dev/null +++ b/t/data/resource/jobspecs/flexible/test004.yaml @@ -0,0 +1,28 @@ +version: 9999 +resources: + - type: slot + count: 1 + label: default + with: + - type: core + count: 8 + - type: memory + count: 2 + - type: or_slot + count: 1 + label: default + with: + - type: core + count: 12 + - type: memory + count: 2 + +# a comment +attributes: + system: + duration: 7200 +tasks: + - command: [ "app" ] + slot: default + count: + per_slot: 1 diff --git a/t/data/resource/jobspecs/flexible/test005.yaml b/t/data/resource/jobspecs/flexible/test005.yaml new file mode 100644 index 000000000..60ebd66c5 --- /dev/null +++ b/t/data/resource/jobspecs/flexible/test005.yaml @@ -0,0 +1,44 @@ +version: 9999 +resources: + - type: or_slot + count: 2 + label: default + with: + - type: node + count: 1 + with: + - type: socket + count: 2 + with: + - type: core + count: 5 + - type: gpu + count: 1 + - type: memory + count: 6 + - type: or_slot + count: 2 + label: default + with: + - type: node + count: 1 + with: + - type: socket + count: 2 + with: + - type: core + count: 5 + - type: gpu + count: 1 + - type: memory + count: 6 + +# a comment +attributes: + system: + duration: 3600 +tasks: + - command: [ "app" ] + slot: default + count: + per_slot: 1 diff --git a/t/data/resource/jobspecs/flexible/test006.yaml b/t/data/resource/jobspecs/flexible/test006.yaml new file mode 100644 index 000000000..d0eb498e1 --- /dev/null +++ b/t/data/resource/jobspecs/flexible/test006.yaml @@ -0,0 +1,21 @@ +version: 9999 +resources: + - type: or_slot + count: 1 + label: default + with: + - type: core + count: 18 + - type: does-not-exist + count: 18 + +# a comment +attributes: + system: + duration: 3600 +tasks: + - command: [ "app" ] + slot: default + count: + per_slot: 1 + diff --git a/t/data/resource/jobspecs/flexible/test007.yaml b/t/data/resource/jobspecs/flexible/test007.yaml new file mode 100644 index 000000000..40521a71f --- /dev/null +++ b/t/data/resource/jobspecs/flexible/test007.yaml @@ -0,0 +1,27 @@ +version: 9999 +resources: + - type: socket + count: 1 + with: + - type: core + count: 8 + - type: memory + count: 2 + - type: or_slot + count: 1 + label: default + with: + - type: core + count: 12 + - type: memory + count: 2 + +# a comment +attributes: + system: + duration: 7200 +tasks: + - command: [ "app" ] + slot: default + count: + per_slot: 1 diff --git a/t/data/resource/jobspecs/flexible/test008.yaml b/t/data/resource/jobspecs/flexible/test008.yaml new file mode 100644 index 000000000..730132bf8 --- /dev/null +++ b/t/data/resource/jobspecs/flexible/test008.yaml @@ -0,0 +1,25 @@ +version: 9999 +resources: + - type: or_slot + count: 1 + label: default + with: + - type: core + count: 1 + - type: or_slot + count: 1 + label: default + with: + - type: core + count: 1 + +# a comment +attributes: + system: + duration: 3600 +tasks: + - command: [ "app" ] + slot: default + count: + per_slot: 1 + diff --git a/t/t3037-resource-flexible.t b/t/t3037-resource-flexible.t new file mode 100755 index 000000000..f4281d975 --- /dev/null +++ b/t/t3037-resource-flexible.t @@ -0,0 +1,76 @@ +#!/bin/sh + +test_description='Test Flexible Scheduling On Medium Machine Configuration in JGF' + +. $(dirname $0)/sharness.sh + +cmd_dir="${SHARNESS_TEST_SRCDIR}/data/resource/commands/flexible" +exp_dir="${SHARNESS_TEST_SRCDIR}/data/resource/expected/flexible" +jgf="${SHARNESS_TEST_SRCDIR}/data/resource/jgfs/tiny.json" +query="../../resource/utilities/resource-query" + +cmds001="${cmd_dir}/cmds01.in" +test001_desc="JGF: allocate 9 jobspecs with flexible scheduling" +test_expect_success "${test001_desc}" ' + sed "s~@TEST_SRCDIR@~${SHARNESS_TEST_SRCDIR}~g" ${cmds001} > cmds001 && + ${query} -L ${jgf} -f jgf -S CA -P high -t 001.R.out < cmds001 && + test_cmp 001.R.out ${exp_dir}/001.R.out +' + +cmds002="${cmd_dir}/cmds02.in" +test002_desc="JGF: allocate jobspecs with valid and invalid or_slots" +test_expect_success "${test002_desc}" ' + sed "s~@TEST_SRCDIR@~${SHARNESS_TEST_SRCDIR}~g" ${cmds002} > cmds002 && + ${query} -L ${jgf} -f jgf -S CA -P high -t 002.R.out < cmds002 && + test_cmp 002.R.out ${exp_dir}/002.R.out +' + +cmds003="${cmd_dir}/cmds03.in" +test003_desc="JGF: allocate jobspecs with pristine jobspec" +test_expect_success "${test003_desc}" ' + sed "s~@TEST_SRCDIR@~${SHARNESS_TEST_SRCDIR}~g" ${cmds003} > cmds003 && + ${query} -L ${jgf} -f jgf -S CA -P high -t 003.R.out < cmds003 && + test_cmp 003.R.out ${exp_dir}/003.R.out +' + +cmds004="${cmd_dir}/cmds04.in" +test004_desc="JGF: try to allocate jobspec with slot and or_slot siblings" +test_expect_success "${test004_desc}" ' + sed "s~@TEST_SRCDIR@~${SHARNESS_TEST_SRCDIR}~g" ${cmds004} > cmds004 && + ${query} -L ${jgf} -f jgf -S CA -P high -t 004.R.out < cmds004 && + test_cmp 004.R.out ${exp_dir}/004.R.out +' + +cmds005="${cmd_dir}/cmds05.in" +test005_desc="JGF: try to allocate jobspec with or_slot count > 1" +test_expect_success "${test005_desc}" ' + sed "s~@TEST_SRCDIR@~${SHARNESS_TEST_SRCDIR}~g" ${cmds005} > cmds005 && + ${query} -L ${jgf} -f jgf -S CA -P high -t 005.R.out < cmds005 && + test_cmp 005.R.out ${exp_dir}/005.R.out +' + +cmds006="${cmd_dir}/cmds06.in" +test006_desc="JGF: try to allocate pristine jobspec with resources that don't exist" +test_expect_success "${test006_desc}" ' + sed "s~@TEST_SRCDIR@~${SHARNESS_TEST_SRCDIR}~g" ${cmds006} > cmds006 && + ${query} -L ${jgf} -f jgf -S CA -P high -t 006.R.out < cmds006 && + test_cmp 006.R.out ${exp_dir}/006.R.out +' + +cmds007="${cmd_dir}/cmds07.in" +test007_desc="JGF: try to allocate jobspec with resource and or_slot siblings" +test_expect_success "${test007_desc}" ' + sed "s~@TEST_SRCDIR@~${SHARNESS_TEST_SRCDIR}~g" ${cmds007} > cmds007 && + ${query} -L ${jgf} -f jgf -S CA -P high -t 007.R.out < cmds007 && + test_cmp 007.R.out ${exp_dir}/007.R.out +' + +cmds008="${cmd_dir}/cmds08.in" +test008_desc="JGF: try to allocate jobspec with identical or_slot configurations" +test_expect_success "${test008_desc}" ' + sed "s~@TEST_SRCDIR@~${SHARNESS_TEST_SRCDIR}~g" ${cmds008} > cmds008 && + ${query} -L ${jgf} -f jgf -S CA -P high -t 008.R.out < cmds008 && + test_cmp 008.R.out ${exp_dir}/008.R.out +' + +test_done From 92bdfb56015f6408cb5adf2a3853c1715c037e14 Mon Sep 17 00:00:00 2001 From: Zeke Morton Date: Wed, 20 Nov 2024 10:09:51 -0800 Subject: [PATCH 4/8] traverser: use map as index for or_config Problem: the or_config map uses a string as an index. Fluxion is moving away from using strings where ever possible. Create a struct and custom hashing function to use the map of resource counts directly as an index. --- resource/traversers/dfu_impl.cpp | 28 ++++++++-------------- resource/traversers/dfu_impl.hpp | 41 ++++++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 20 deletions(-) diff --git a/resource/traversers/dfu_impl.cpp b/resource/traversers/dfu_impl.cpp index b03e4a1c2..ee47f3a5e 100644 --- a/resource/traversers/dfu_impl.cpp +++ b/resource/traversers/dfu_impl.cpp @@ -752,22 +752,18 @@ int dfu_impl_t::dom_slot (const jobmeta_t &meta, return (qual_num_slots) ? 0 : -1; } -std::tuple dfu_impl_t::select_or_config ( +std::tuple, int, int> dfu_impl_t::select_or_config ( const std::vector &slots, std::map resource_counts, unsigned int nslots, - std::map> &or_config) + std::unordered_map, int, int>, Hash> &or_config) { int best = -1; int i = -1; - std::string index = ""; - - // generate or_config index based on resource counts - for (auto it : resource_counts) - index = index + std::to_string (it.second) + " "; + Key index = Key (resource_counts); // if available, use precomputed result - auto it = or_config.find (index); + auto it = or_config.find (resource_counts); if (it != or_config.end ()) return it->second; @@ -776,7 +772,6 @@ std::tuple dfu_impl_t::select_or_config ( ++i; bool match = true; std::map updated_counts; - std::string updated_index = ""; updated_counts = resource_counts; // determine if there are enough resources to match with this or_slot @@ -792,14 +787,10 @@ std::tuple dfu_impl_t::select_or_config ( if (!match) continue; - // find the best score after using resources from this or_slot - for (auto it : updated_counts) - updated_index = updated_index + std::to_string (it.second) + " "; - test = std::get<1> (select_or_config (slots, updated_counts, nslots, or_config)); if (best < test) { best = test; - or_config[index] = std::make_tuple (updated_index, best + 1, i); + or_config[index] = std::make_tuple (updated_counts, best + 1, i); } } @@ -807,7 +798,8 @@ std::tuple dfu_impl_t::select_or_config ( // score represents the total number of or_slots that can be scheduled // with optimal selection of or_slots if (best < 0) { - or_config[index] = std::make_tuple ("", best + 1, -1); + std::map empty; + or_config[index] = std::make_tuple (empty, best + 1, -1); } return or_config[index]; } @@ -826,8 +818,8 @@ int dfu_impl_t::dom_or_slot (const jobmeta_t &meta, const subsystem_t &dom = m_match->dom_subsystem (); std::unordered_set edges_used; scoring_api_t dfu_slot; - std::map> or_config; - std::tuple current_config; + std::unordered_map, int, int>, Hash> or_config; + std::tuple, int, int> current_config; // collect a set of all resource types in the or_slots to get resource // counts. This does not work well with non leaf vertex resources because @@ -899,7 +891,7 @@ int dfu_impl_t::dom_or_slot (const jobmeta_t &meta, edg_group.exclusive = 1; edg_group_vector.push_back (edg_group); - current_config = or_config[std::get<0> (current_config)]; + current_config = or_config[Key (std::get<0> (current_config))]; } for (auto &edg_group : edg_group_vector) dfu.add (dom, or_slot_rt, edg_group); diff --git a/resource/traversers/dfu_impl.hpp b/resource/traversers/dfu_impl.hpp index 4ff06a321..2dba54531 100644 --- a/resource/traversers/dfu_impl.hpp +++ b/resource/traversers/dfu_impl.hpp @@ -346,6 +346,41 @@ class dfu_impl_t { bool get_eff_exclusive (bool x, bool mod) const; unsigned get_eff_needs (unsigned needs, unsigned size, bool mod) const; + // struct to convert map of resources counts to an index + struct Key { + Key (); + Key (const std::map &c) + { + counts = c; + }; + + std::map counts; + + bool operator== (const Key &other) const + { + if (counts.size () != other.counts.size ()) + return false; + + for (auto &it : counts) { + if (it.second != other.counts.at (it.first)) + return false; + } + return true; + } + }; + + // Custom hashing function for resource counts map + struct Hash { + std::size_t operator() (const Key &k) const + { + std::size_t seed = 0; + for (auto &it : k.counts) { + boost::hash_combine (seed, it.second); + } + return seed; + } + }; + /*! Various pruning methods */ int by_avail (const jobmeta_t &meta, @@ -467,11 +502,13 @@ class dfu_impl_t { bool prestine, bool *excl, scoring_api_t &dfu); - std::tuple select_or_config ( + // std::size_t hash_value(std::map counts); + std::tuple, int, int> select_or_config ( const std::vector &slots, std::map resource_counts, unsigned int nslots, - std::map> &or_config); + std::unordered_map, int, int>, Hash> + &or_config); int dom_or_slot (const jobmeta_t &meta, vtx_t u, const std::vector &resources, From f1aa1673d66c76b8902d8e1e1e453360b7ab1ece Mon Sep 17 00:00:00 2001 From: Zeke Morton Date: Wed, 8 Jan 2025 15:55:19 -0800 Subject: [PATCH 5/8] traverser: move or slots to dedicated flexible traverser Problem: a flexible traverser with or_slots is only one posible traverser policy. There should be a frame work to select different traverser based on the desired policy. Create a new flexible traverser class that inherits from dfu_traverser_t and overload functions where or_slot logic goes and implemnt slots as or_slots. Remove or_slots from dfu_impl_t. --- resource/CMakeLists.txt | 2 + resource/traversers/dfu_flexible.cpp | 324 +++++++++++++++++++++++++++ resource/traversers/dfu_flexible.hpp | 119 ++++++++++ resource/traversers/dfu_impl.cpp | 228 +------------------ resource/traversers/dfu_impl.hpp | 79 +------ 5 files changed, 461 insertions(+), 291 deletions(-) create mode 100644 resource/traversers/dfu_flexible.cpp create mode 100644 resource/traversers/dfu_flexible.hpp diff --git a/resource/CMakeLists.txt b/resource/CMakeLists.txt index 625be874c..c75f03da3 100644 --- a/resource/CMakeLists.txt +++ b/resource/CMakeLists.txt @@ -29,6 +29,7 @@ set(RESOURCE_HEADERS schema/ephemeral.hpp traversers/dfu.hpp traversers/dfu_impl.hpp + traversers/dfu_flexible.hpp policies/base/dfu_match_cb.hpp policies/base/matcher.hpp readers/resource_namespace_remapper.hpp @@ -62,6 +63,7 @@ add_library(resource STATIC schema/ephemeral.cpp traversers/dfu.cpp traversers/dfu_impl.cpp + traversers/dfu_flexible.cpp traversers/dfu_impl_update.cpp policies/base/dfu_match_cb.cpp policies/base/matcher.cpp diff --git a/resource/traversers/dfu_flexible.cpp b/resource/traversers/dfu_flexible.cpp new file mode 100644 index 000000000..510c157a3 --- /dev/null +++ b/resource/traversers/dfu_flexible.cpp @@ -0,0 +1,324 @@ +/*****************************************************************************\ + * Copyright 2024 Lawrence Livermore National Security, LLC + * (c.f. AUTHORS, NOTICE.LLNS, LICENSE) + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * SPDX-License-Identifier: LGPL-3.0 +\*****************************************************************************/ + +extern "C" { +#if HAVE_CONFIG_H +#include "config.h" +#endif +} + +#include "resource/traversers/dfu_flexible.hpp" + +using namespace Flux::Jobspec; +using namespace Flux::resource_model; +using namespace Flux::resource_model::detail; + +int dfu_flexible_t::match (vtx_t u, + const std::vector &resources, + const Resource **slot_resource, + unsigned int *nslots, + const Resource **match_resource, + const std::vector **slot_resources) +{ + int rc = -1; + bool matched = false, or_matched = false; + for (auto &resource : resources) { + if ((*m_graph)[u].type == resource.type) { + // Limitations of DFU traverser: jobspec must not + // have same type at same level Please read utilities/README.md + if (matched || or_matched) + goto ret; + *match_resource = &resource; + if (!resource.with.empty ()) { + for (auto &c_resource : resource.with) { + if (c_resource.type == slot_rt) { + *slot_resource = &c_resource; + *nslots = m_match->calc_effective_max (c_resource); + } + } + } + matched = true; + } else if (resource.type == slot_rt) { + // Limitations of DFU traverser: jobspec must not + // have same type at same level except for or_slot. + if (matched) + goto ret; + *slot_resources = &resources; + // This value is not well defined. In this state, nslots is + // determined by the last listed or_slot sibling in the jobspec. + *nslots = m_match->calc_effective_max (resource); + or_matched = true; + } + } + rc = 0; + +ret: + return rc; +} + +const std::vector &dfu_flexible_t::test (vtx_t u, + const std::vector &resources, + bool &pristine, + unsigned int &nslots, + match_kind_t &spec) +{ + /* Note on the purpose of pristine: we differentiate two similar but + * distinct cases with this parameter. + * Jobspec is allowed to omit the prefix so you can have a spec like + * socket[1]->core[2] which will match + * cluster[1]->node[1]->socket[2]->core[22]. + * For this case, when you visit the "node" resource vertex, the next + * Jobspec resource that should be used at the next recursion should + * be socket[1]. And we enable this if pristine is true. + * But then once the first match is made, any mismatch afterwards + * should result in a match failure. For example, + * socket[1]->core[2] must fail to match + * cluster[1]->socket[1]->numanode[1]->core[22]. + * pristine is used to detect this case. + */ + bool slot = true; + const std::vector *ret = &resources; + const Resource *slot_resources = NULL; + const Resource *match_resources = NULL; + const std::vector *slot_or_resources = NULL; + if (match (u, resources, &slot_resources, &nslots, &match_resources, &slot_or_resources) < 0) { + m_err_msg += __FUNCTION__; + m_err_msg += ": siblings in jobspec request same resource type "; + m_err_msg += ": " + (*m_graph)[u].type + ".\n"; + spec = match_kind_t::NONE_MATCH; + goto done; + } + if ((slot_or_resources)) { + // set default spec in case no match is found + spec = pristine ? match_kind_t::PRISTINE_NONE_MATCH : match_kind_t::NONE_MATCH; + + for (Resource r : *slot_or_resources) { + if ((slot_match (u, &r))) { + spec = match_kind_t::SLOT_MATCH; + pristine = false; + ret = slot_or_resources; + } + } + } else if (match_resources) { + spec = match_kind_t::RESOURCE_MATCH; + pristine = false; + ret = &(match_resources->with); + } else { + spec = pristine ? match_kind_t::PRISTINE_NONE_MATCH : match_kind_t::NONE_MATCH; + } + +done: + return *ret; +} + +/* Same as above except that lowest is unorder_map */ +int dfu_flexible_t::min_if (subsystem_t subsystem, + resource_type_t type, + unsigned int counts, + std::unordered_map &lowest) +{ + int rc = -1; + if (m_match->is_pruning_type (subsystem, type)) { + if (lowest.find (type) == lowest.end ()) + lowest[type] = counts; + else if (lowest[type] > counts) + lowest[type] = counts; + rc = 0; + } + return rc; +} + +void dfu_flexible_t::prime_jobspec (std::vector &resources, + std::unordered_map &to_parent) +{ + subsystem_t subsystem = m_match->dom_subsystem (); + for (auto &resource : resources) { + // If the resource is requested as exclusive in the + // jobspec, add it to the matcher's exclusive resource + // set. This ensures that the full resource set (which + // includes shadow resources) is emitted. + if (resource.exclusive == Jobspec::tristate_t::TRUE) + m_match->add_exclusive_resource_type (resource.type); + // Use minimum requirement because you don't want to prune search + // as far as a subtree satisfies the minimum requirement + accum_if (subsystem, resource.type, resource.count.min, to_parent); + prime_jobspec (resource.with, resource.user_data); + + // Or slots should use a minimum of values rather than an accumulation + // otherwise possible matches may be filtered out + if (resource.type == or_slot_rt) { + for (auto &aggregate : resource.user_data) { + min_if (subsystem, + aggregate.first, + resource.count.min * aggregate.second, + to_parent); + } + } else { + for (auto &aggregate : resource.user_data) { + accum_if (subsystem, + aggregate.first, + resource.count.min * aggregate.second, + to_parent); + } + } + } +} + +std::tuple, int, int> dfu_flexible_t::select_or_config ( + const std::vector &slots, + std::map resource_counts, + unsigned int nslots, + std::unordered_map, int, int>, Hash> &or_config) +{ + int best = -1; + int i = -1; + Key index = Key (resource_counts); + + // if available, use precomputed result + auto it = or_config.find (resource_counts); + if (it != or_config.end ()) + return it->second; + + for (auto slot : slots) { + int test; + ++i; + bool match = true; + std::map updated_counts; + updated_counts = resource_counts; + + // determine if there are enough resources to match with this or_slot + for (auto slot_elem : slot.with) { + unsigned int qc = resource_counts[slot_elem.type]; + unsigned int count = m_match->calc_count (slot_elem, qc); + if (count <= 0) { + match = false; + break; + } + updated_counts[slot_elem.type] = updated_counts[slot_elem.type] - count; + } + if (!match) + continue; + + test = std::get<1> (select_or_config (slots, updated_counts, nslots, or_config)); + if (best < test) { + best = test; + or_config[index] = std::make_tuple (updated_counts, best + 1, i); + } + } + + // if there are no matches, set default score of 0 + // score represents the total number of or_slots that can be scheduled + // with optimal selection of or_slots + if (best < 0) { + std::map empty; + or_config[index] = std::make_tuple (empty, best + 1, -1); + } + return or_config[index]; +} + +int dfu_flexible_t::dom_slot (const jobmeta_t &meta, + vtx_t u, + const std::vector &slots, + unsigned int nslots, + bool pristine, + bool *excl, + scoring_api_t &dfu) +{ + int rc; + bool x_inout = true; + unsigned int qual_num_slots = 0; + std::vector edg_group_vector; + const subsystem_t &dom = m_match->dom_subsystem (); + std::unordered_set edges_used; + scoring_api_t dfu_slot; + std::unordered_map, int, int>, Hash> or_config; + std::tuple, int, int> current_config; + + // collect a set of all resource types in the or_slots to get resource + // counts. This does not work well with non leaf vertex resources because + // it cannot distinguish beyond type. This may be resolveable if graph + // coloring is removed during the selection process. + std::vector slot_resource_union; + std::map resource_types; + for (auto &slot : slots) { + for (auto r : slot.with) { + if (resource_types.find (r.type) == resource_types.end ()) { + resource_types[r.type] = 0; + slot_resource_union.push_back (r); + } + } + } + + if ((rc = explore (meta, + u, + dom, + slot_resource_union, + pristine, + &x_inout, + visit_t::DFV, + dfu_slot, + nslots)) + != 0) + goto done; + if ((rc = m_match->dom_finish_slot (dom, dfu_slot)) != 0) + goto done; + + for (auto &it : resource_types) { + it.second = dfu_slot.qualified_count (dom, it.first); + } + + // calculate the ideal or_slot config for avail resources. + // tuple is (key to next best option, current score, index of current best or_slot) + current_config = select_or_config (slots, resource_types, nslots, or_config); + + qual_num_slots = std::get<1> (current_config); + for (unsigned int i = 0; i < qual_num_slots; ++i) { + auto slot_index = std::get<2> (current_config); + eval_egroup_t edg_group; + int64_t score = MATCH_MET; + + // use calculated index to determine which or_slot type to use + for (auto &slot_elem : slots[slot_index].with) { + unsigned int j = 0; + unsigned int qc = dfu_slot.qualified_count (dom, slot_elem.type); + unsigned int count = m_match->calc_count (slot_elem, qc); + while (j < count) { + auto egroup_i = dfu_slot.eval_egroups_iter_next (dom, slot_elem.type); + if (egroup_i == dfu_slot.eval_egroups_end (dom, slot_elem.type)) { + m_err_msg += __FUNCTION__; + m_err_msg += ": not enough slots.\n"; + qual_num_slots = 0; + goto done; + } + eval_edg_t ev_edg ((*egroup_i).edges[0].count, + (*egroup_i).edges[0].count, + 1, + (*egroup_i).edges[0].edge); + score += (*egroup_i).score; + edg_group.edges.push_back (ev_edg); + j += (*egroup_i).edges[0].count; + } + } + edg_group.score = score; + edg_group.count = 1; + edg_group.exclusive = 1; + edg_group_vector.push_back (edg_group); + + current_config = or_config[Key (std::get<0> (current_config))]; + } + for (auto &edg_group : edg_group_vector) + dfu.add (dom, or_slot_rt, edg_group); + +done: + return (qual_num_slots) ? 0 : -1; +} +/* + * vi:tabstop=4 shiftwidth=4 expandtab + */ diff --git a/resource/traversers/dfu_flexible.hpp b/resource/traversers/dfu_flexible.hpp new file mode 100644 index 000000000..deabec53a --- /dev/null +++ b/resource/traversers/dfu_flexible.hpp @@ -0,0 +1,119 @@ +/*****************************************************************************\ + * Copyright 2024 Lawrence Livermore National Security, LLC + * (c.f. AUTHORS, NOTICE.LLNS, LICENSE) + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * SPDX-License-Identifier: LGPL-3.0 +\*****************************************************************************/ + +#ifndef DFU_FLEXIBLE_HPP +#define DFU_FLEXIBLE_HPP + +#include "resource/traversers/dfu.hpp" + +using namespace Flux::resource_model::detail; +namespace Flux { +namespace resource_model { + +class dfu_flexible_t : public dfu_traverser_t { + // struct to convert map of resources counts to an index + struct Key { + Key (); + Key (const std::map &c) + { + counts = c; + }; + + std::map counts; + + bool operator== (const Key &other) const + { + if (counts.size () != other.counts.size ()) + return false; + + for (auto &it : counts) { + if (it.second != other.counts.at (it.first)) + return false; + } + return true; + } + }; + + // Custom hashing function for resource counts map + struct Hash { + std::size_t operator() (const Key &k) const + { + std::size_t seed = 0; + for (auto &it : k.counts) { + boost::hash_combine (seed, it.second); + } + return seed; + } + }; + + int match (vtx_t u, + const std::vector &resources, + const Jobspec::Resource **slot_resource, + unsigned int *nslots, + const Jobspec::Resource **match_resource, + const std::vector **slot_resources); + const std::vector &test (vtx_t u, + const std::vector &resources, + bool &prestine, + unsigned int &nslots, + match_kind_t &ko); + std::tuple, int, int> select_or_config ( + const std::vector &slots, + std::map resource_counts, + unsigned int nslots, + std::unordered_map, int, int>, Hash> + &or_config); + + /*! Find min count if type matches with one of the resource + * types used in the scheduler-driven aggregate update (SDAU) scheme. + * dfu_match_cb_t provides an interface to configure what types are used + * for SDAU scheme. + */ + int min_if (subsystem_t subsystem, + resource_type_t type, + unsigned int count, + std::unordered_map &lowest); + + int dom_slot (const jobmeta_t &meta, + vtx_t u, + const std::vector &resources, + unsigned int nslots, + bool prestine, + bool *excl, + scoring_api_t &dfu); + + /*! Prime the resource section of the jobspec. Aggregate configured + * subtree resources into jobspec's user_data. For example, + * cluster[1]->rack[2]->node[4]->socket[1]->core[2] + * with socket and core types configured to be tracked will be augmented + * at the end of priming as: + * cluster[1](core:16)->rack[2](core:8)->node[4](core:2)-> + * socket[1](core:2)->core[2] + * + * The subtree aggregate information is used to prune unnecessary + * graph traversals + * + * \param resources Resource request vector. + * \param[out] to_parent + * output aggregates on the subtree. + * \return none. + */ + void prime_jobspec (std::vector &resources, + std::unordered_map &to_parent); +}; + +} // namespace resource_model +} // namespace Flux + +#endif // DFU_TRAVERSE_HPP + +/* + * vi:tabstop=4 shiftwidth=4 expandtab + */ diff --git a/resource/traversers/dfu_impl.cpp b/resource/traversers/dfu_impl.cpp index ee47f3a5e..0c6683932 100644 --- a/resource/traversers/dfu_impl.cpp +++ b/resource/traversers/dfu_impl.cpp @@ -259,49 +259,33 @@ int dfu_impl_t::match (vtx_t u, const std::vector &resources, const Resource **slot_resource, unsigned int *nslots, - const Resource **match_resource, - const std::vector **slot_resources) + const Resource **match_resource) { int rc = -1; - bool matched = false, or_matched = false; + bool matched = false; for (auto &resource : resources) { if ((*m_graph)[u].type == resource.type) { // Limitations of DFU traverser: jobspec must not // have same type at same level Please read utilities/README.md - if (matched || or_matched) + if (matched == true) goto ret; *match_resource = &resource; if (!resource.with.empty ()) { - for (auto &c_resource : resource.with) { - if (c_resource.type == or_slot_rt) { - *slot_resources = &resource.with; - *nslots = m_match->calc_effective_max (c_resource); - } + for (auto &c_resource : resource.with) if (c_resource.type == slot_rt) { *slot_resource = &c_resource; *nslots = m_match->calc_effective_max (c_resource); } - } } matched = true; } else if (resource.type == slot_rt) { // Limitations of DFU traverser: jobspec must not // have same type at same level Please read utilities/README.md - if (matched || or_matched) + if (matched == true) goto ret; *slot_resource = &resource; *nslots = m_match->calc_effective_max (resource); matched = true; - } else if (resource.type == or_slot_rt) { - // Limitations of DFU traverser: jobspec must not - // have same type at same level except for or_slot. - if (matched) - goto ret; - *slot_resources = &resources; - // This value is not well defined. In this state, nslots is - // determined by the last listed or_slot sibling in the jobspec. - *nslots = m_match->calc_effective_max (resource); - or_matched = true; } } rc = 0; @@ -356,26 +340,14 @@ const std::vector &dfu_impl_t::test (vtx_t u, const std::vector *ret = &resources; const Resource *slot_resources = NULL; const Resource *match_resources = NULL; - const std::vector *slot_or_resources = NULL; - if (match (u, resources, &slot_resources, &nslots, &match_resources, &slot_or_resources) < 0) { + if (match (u, resources, &slot_resources, &nslots, &match_resources) < 0) { m_err_msg += __FUNCTION__; m_err_msg += ": siblings in jobspec request same resource type "; m_err_msg += ": " + (*m_graph)[u].type + ".\n"; spec = match_kind_t::NONE_MATCH; goto done; } - if ((slot_or_resources)) { - // set default spec in case no match is found - spec = pristine ? match_kind_t::PRISTINE_NONE_MATCH : match_kind_t::NONE_MATCH; - - for (Resource r : *slot_or_resources) { - if ((slot_match (u, &r))) { - spec = match_kind_t::OR_SLOT_MATCH; - pristine = false; - ret = slot_or_resources; - } - } - } else if ((slot = slot_match (u, slot_resources))) { + if ((slot = slot_match (u, slot_resources))) { spec = match_kind_t::SLOT_MATCH; pristine = false; ret = &(slot_resources->with); @@ -425,23 +397,6 @@ int dfu_impl_t::accum_if (subsystem_t subsystem, return rc; } -/* Same as above except that lowest is unorder_map */ -int dfu_impl_t::min_if (subsystem_t subsystem, - resource_type_t type, - unsigned int counts, - std::unordered_map &lowest) -{ - int rc = -1; - if (m_match->is_pruning_type (subsystem, type)) { - if (lowest.find (type) == lowest.end ()) - lowest[type] = counts; - else if (lowest[type] > counts) - lowest[type] = counts; - rc = 0; - } - return rc; -} - int dfu_impl_t::prime_exp (subsystem_t subsystem, vtx_t u, std::map &dfv) { int rc = 0; @@ -752,154 +707,6 @@ int dfu_impl_t::dom_slot (const jobmeta_t &meta, return (qual_num_slots) ? 0 : -1; } -std::tuple, int, int> dfu_impl_t::select_or_config ( - const std::vector &slots, - std::map resource_counts, - unsigned int nslots, - std::unordered_map, int, int>, Hash> &or_config) -{ - int best = -1; - int i = -1; - Key index = Key (resource_counts); - - // if available, use precomputed result - auto it = or_config.find (resource_counts); - if (it != or_config.end ()) - return it->second; - - for (auto slot : slots) { - int test; - ++i; - bool match = true; - std::map updated_counts; - updated_counts = resource_counts; - - // determine if there are enough resources to match with this or_slot - for (auto slot_elem : slot.with) { - unsigned int qc = resource_counts[slot_elem.type]; - unsigned int count = m_match->calc_count (slot_elem, qc); - if (count <= 0) { - match = false; - break; - } - updated_counts[slot_elem.type] = updated_counts[slot_elem.type] - count; - } - if (!match) - continue; - - test = std::get<1> (select_or_config (slots, updated_counts, nslots, or_config)); - if (best < test) { - best = test; - or_config[index] = std::make_tuple (updated_counts, best + 1, i); - } - } - - // if there are no matches, set default score of 0 - // score represents the total number of or_slots that can be scheduled - // with optimal selection of or_slots - if (best < 0) { - std::map empty; - or_config[index] = std::make_tuple (empty, best + 1, -1); - } - return or_config[index]; -} -int dfu_impl_t::dom_or_slot (const jobmeta_t &meta, - vtx_t u, - const std::vector &slots, - unsigned int nslots, - bool pristine, - bool *excl, - scoring_api_t &dfu) -{ - int rc; - bool x_inout = true; - unsigned int qual_num_slots = 0; - std::vector edg_group_vector; - const subsystem_t &dom = m_match->dom_subsystem (); - std::unordered_set edges_used; - scoring_api_t dfu_slot; - std::unordered_map, int, int>, Hash> or_config; - std::tuple, int, int> current_config; - - // collect a set of all resource types in the or_slots to get resource - // counts. This does not work well with non leaf vertex resources because - // it cannot distinguish beyond type. This may be resolveable if graph - // coloring is removed during the selection process. - std::vector slot_resource_union; - std::map resource_types; - for (auto &slot : slots) { - for (auto r : slot.with) { - if (resource_types.find (r.type) == resource_types.end ()) { - resource_types[r.type] = 0; - slot_resource_union.push_back (r); - } - } - } - - if ((rc = explore (meta, - u, - dom, - slot_resource_union, - pristine, - &x_inout, - visit_t::DFV, - dfu_slot, - nslots)) - != 0) - goto done; - if ((rc = m_match->dom_finish_slot (dom, dfu_slot)) != 0) - goto done; - - for (auto &it : resource_types) { - it.second = dfu_slot.qualified_count (dom, it.first); - } - - // calculate the ideal or_slot config for avail resources. - // tuple is (key to next best option, current score, index of current best or_slot) - current_config = select_or_config (slots, resource_types, nslots, or_config); - - qual_num_slots = std::get<1> (current_config); - for (unsigned int i = 0; i < qual_num_slots; ++i) { - auto slot_index = std::get<2> (current_config); - eval_egroup_t edg_group; - int64_t score = MATCH_MET; - - // use calculated index to determine which or_slot type to use - for (auto &slot_elem : slots[slot_index].with) { - unsigned int j = 0; - unsigned int qc = dfu_slot.qualified_count (dom, slot_elem.type); - unsigned int count = m_match->calc_count (slot_elem, qc); - while (j < count) { - auto egroup_i = dfu_slot.eval_egroups_iter_next (dom, slot_elem.type); - if (egroup_i == dfu_slot.eval_egroups_end (dom, slot_elem.type)) { - m_err_msg += __FUNCTION__; - m_err_msg += ": not enough slots.\n"; - qual_num_slots = 0; - goto done; - } - eval_edg_t ev_edg ((*egroup_i).edges[0].count, - (*egroup_i).edges[0].count, - 1, - (*egroup_i).edges[0].edge); - score += (*egroup_i).score; - edg_group.edges.push_back (ev_edg); - j += (*egroup_i).edges[0].count; - } - } - edg_group.score = score; - edg_group.count = 1; - edg_group.exclusive = 1; - edg_group_vector.push_back (edg_group); - - current_config = or_config[Key (std::get<0> (current_config))]; - } - for (auto &edg_group : edg_group_vector) - dfu.add (dom, or_slot_rt, edg_group); - -done: - return (qual_num_slots) ? 0 : -1; -} - int dfu_impl_t::dom_dfv (const jobmeta_t &meta, vtx_t u, const std::vector &resources, @@ -929,8 +736,6 @@ int dfu_impl_t::dom_dfv (const jobmeta_t &meta, (*m_graph)[u].idata.colors[dom] = m_color.gray (); if (sm == match_kind_t::SLOT_MATCH) dom_slot (meta, u, next, nslots, check_pres, &x_inout, dfu); - else if (sm == match_kind_t::OR_SLOT_MATCH) - dom_or_slot (meta, u, next, nslots, check_pres, &x_inout, dfu); else dom_exp (meta, u, next, check_pres, &x_inout, dfu); *excl = x_in; @@ -1349,23 +1154,8 @@ void dfu_impl_t::prime_jobspec (std::vector &resources, // as far as a subtree satisfies the minimum requirement accum_if (subsystem, resource.type, resource.count.min, to_parent); prime_jobspec (resource.with, resource.user_data); - - // Or slots should use a minimum of values rather than an accumulation - // otherwise possible matches may be filtered out - if (resource.type == or_slot_rt) { - for (auto &aggregate : resource.user_data) { - min_if (subsystem, - aggregate.first, - resource.count.min * aggregate.second, - to_parent); - } - } else { - for (auto &aggregate : resource.user_data) { - accum_if (subsystem, - aggregate.first, - resource.count.min * aggregate.second, - to_parent); - } + for (auto &aggregate : resource.user_data) { + accum_if (subsystem, aggregate.first, resource.count.min * aggregate.second, to_parent); } } } diff --git a/resource/traversers/dfu_impl.hpp b/resource/traversers/dfu_impl.hpp index 2dba54531..994213f90 100644 --- a/resource/traversers/dfu_impl.hpp +++ b/resource/traversers/dfu_impl.hpp @@ -32,13 +32,7 @@ namespace detail { enum class visit_t { DFV, UPV }; -enum class match_kind_t { - RESOURCE_MATCH, - SLOT_MATCH, - OR_SLOT_MATCH, - NONE_MATCH, - PRISTINE_NONE_MATCH -}; +enum class match_kind_t { RESOURCE_MATCH, SLOT_MATCH, NONE_MATCH, PRISTINE_NONE_MATCH }; struct jobmeta_t { enum class alloc_type_t : int { @@ -103,7 +97,7 @@ struct jobmeta_t { return 0; } - private: + protected: bool m_queue_set = false; std::string m_queue = ""; }; @@ -330,10 +324,10 @@ class dfu_impl_t { */ int mark (std::set &ranks, resource_pool_t::status_t status); - private: + protected: /************************************************************************ * * - * Private Match and Util API * + * Protected Match and Util API * * * ************************************************************************/ const std::string level () const; @@ -346,41 +340,6 @@ class dfu_impl_t { bool get_eff_exclusive (bool x, bool mod) const; unsigned get_eff_needs (unsigned needs, unsigned size, bool mod) const; - // struct to convert map of resources counts to an index - struct Key { - Key (); - Key (const std::map &c) - { - counts = c; - }; - - std::map counts; - - bool operator== (const Key &other) const - { - if (counts.size () != other.counts.size ()) - return false; - - for (auto &it : counts) { - if (it.second != other.counts.at (it.first)) - return false; - } - return true; - } - }; - - // Custom hashing function for resource counts map - struct Hash { - std::size_t operator() (const Key &k) const - { - std::size_t seed = 0; - for (auto &it : k.counts) { - boost::hash_combine (seed, it.second); - } - return seed; - } - }; - /*! Various pruning methods */ int by_avail (const jobmeta_t &meta, @@ -413,8 +372,7 @@ class dfu_impl_t { const std::vector &resources, const Jobspec::Resource **slot_resource, unsigned int *nslots, - const Jobspec::Resource **match_resource, - const std::vector **slot_resources); + const Jobspec::Resource **match_resource); bool slot_match (vtx_t u, const Jobspec::Resource *slot_resource); const std::vector &test (vtx_t u, const std::vector &resources, @@ -437,16 +395,6 @@ class dfu_impl_t { unsigned int count, std::unordered_map &accum); - /*! Find min count if type matches with one of the resource - * types used in the scheduler-driven aggregate update (SDAU) scheme. - * dfu_match_cb_t provides an interface to configure what types are used - * for SDAU scheme. - */ - int min_if (subsystem_t subsystem, - resource_type_t type, - unsigned int count, - std::unordered_map &lowest); - // Explore out-edges for priming the subtree plans int prime_exp (subsystem_t subsystem, vtx_t u, std::map &dfv); @@ -503,19 +451,6 @@ class dfu_impl_t { bool *excl, scoring_api_t &dfu); // std::size_t hash_value(std::map counts); - std::tuple, int, int> select_or_config ( - const std::vector &slots, - std::map resource_counts, - unsigned int nslots, - std::unordered_map, int, int>, Hash> - &or_config); - int dom_or_slot (const jobmeta_t &meta, - vtx_t u, - const std::vector &resources, - unsigned int nslots, - bool prestine, - bool *excl, - scoring_api_t &dfu); int dom_exp (const jobmeta_t &meta, vtx_t u, const std::vector &resources, @@ -556,7 +491,7 @@ class dfu_impl_t { /************************************************************************ * * - * Private Update/Emit/Remove API * + * Protected Update/Emit/Remove API * * * ************************************************************************/ // Emit matched resource set @@ -643,7 +578,7 @@ class dfu_impl_t { /************************************************************************ * * - * Private Member Data * + * Protected Member Data * * * ************************************************************************/ color_t m_color; From 27acbde26e4171586e280cfc6cc14628a53e8782 Mon Sep 17 00:00:00 2001 From: Zeke Morton Date: Wed, 29 Jan 2025 16:32:47 -0800 Subject: [PATCH 6/8] traverser: add policy factory Problem: there is no function to create a traverser by policy Add a function that takes policy string and returns traverser. --- resource/CMakeLists.txt | 2 + .../dfu_traverser_policy_factory.cpp | 54 +++++++++++++++++++ .../dfu_traverser_policy_factory.hpp | 39 ++++++++++++++ 3 files changed, 95 insertions(+) create mode 100644 resource/traversers/dfu_traverser_policy_factory.cpp create mode 100644 resource/traversers/dfu_traverser_policy_factory.hpp diff --git a/resource/CMakeLists.txt b/resource/CMakeLists.txt index c75f03da3..269ad7b79 100644 --- a/resource/CMakeLists.txt +++ b/resource/CMakeLists.txt @@ -30,6 +30,7 @@ set(RESOURCE_HEADERS traversers/dfu.hpp traversers/dfu_impl.hpp traversers/dfu_flexible.hpp + traversers/dfu_traverser_policy_factory.hpp policies/base/dfu_match_cb.hpp policies/base/matcher.hpp readers/resource_namespace_remapper.hpp @@ -64,6 +65,7 @@ add_library(resource STATIC traversers/dfu.cpp traversers/dfu_impl.cpp traversers/dfu_flexible.cpp + traversers/dfu_traverser_policy_factory.cpp traversers/dfu_impl_update.cpp policies/base/dfu_match_cb.cpp policies/base/matcher.cpp diff --git a/resource/traversers/dfu_traverser_policy_factory.cpp b/resource/traversers/dfu_traverser_policy_factory.cpp new file mode 100644 index 000000000..d00d055dc --- /dev/null +++ b/resource/traversers/dfu_traverser_policy_factory.cpp @@ -0,0 +1,54 @@ +/*****************************************************************************\ + * Copyright 2025 Lawrence Livermore National Security, LLC + * (c.f. AUTHORS, NOTICE.LLNS, LICENSE) + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * SPDX-License-Identifier: LGPL-3.0 +\*****************************************************************************/ + +extern "C" { +#if HAVE_CONFIG_H +#include +#endif +} + +#include +#include "resource/traversers/dfu_traverser_policy_factory.hpp" + +namespace Flux { +namespace resource_model { + +bool known_traverser_policy (const std::string &policy) +{ + bool rc = true; + if (policy != SIMPLE && policy != FLEXIBLE) + rc = false; + + return rc; +} + +std::shared_ptr create_traverser (const std::string &policy) +{ + std::shared_ptr traverser = nullptr; + try { + if (policy == FLEXIBLE) { + traverser = std::make_shared (); + } else if (policy == SIMPLE) { + traverser = std::make_shared (); + } + } catch (std::bad_alloc &e) { + errno = ENOMEM; + traverser = nullptr; + } + + return traverser; +} + +} // namespace resource_model +} // namespace Flux + +/* + * vi:tabstop=4 shiftwidth=4 expandtab + */ diff --git a/resource/traversers/dfu_traverser_policy_factory.hpp b/resource/traversers/dfu_traverser_policy_factory.hpp new file mode 100644 index 000000000..9f69ecf30 --- /dev/null +++ b/resource/traversers/dfu_traverser_policy_factory.hpp @@ -0,0 +1,39 @@ +/*****************************************************************************\ + * Copyright 2025 Lawrence Livermore National Security, LLC + * (c.f. AUTHORS, NOTICE.LLNS, LICENSE) + * + * This file is part of the Flux resource manager framework. + * For details, see https://github.com/flux-framework. + * + * SPDX-License-Identifier: LGPL-3.0 +\*****************************************************************************/ + +#ifndef DFU_TRAVERSER_POLICY_FACTORY_HPP +#define DFU_TRAVERSER_POLICY_FACTORY_HPP + +#include +#include +#include "resource/traversers/dfu.hpp" +#include "resource/traversers/dfu_flexible.hpp" + +namespace Flux { +namespace resource_model { + +const std::string SIMPLE = "simple"; +const std::string FLEXIBLE = "flexible"; + +bool known_traverser_policy (const std::string &policy); + +/*! Factory method for creating a matching callback + * object, representing a matching policy. + */ +std::shared_ptr create_traverser (const std::string &policy); + +} // namespace resource_model +} // namespace Flux + +#endif // DFU_TRAVERSER_POLICY_FACTORY_HPP + +/* + * vi:tabstop=4 shiftwidth=4 expandtab + */ From c0652619c84a786b3c379de79b3e75c8b233b072 Mon Sep 17 00:00:00 2001 From: Zeke Morton Date: Wed, 29 Jan 2025 16:39:36 -0800 Subject: [PATCH 7/8] rq: add traverser policy option Problem: resource query does not allow for a command line option to select the traverser policy. Add and implement new option. --- resource/utilities/command.hpp | 23 ++++++++++++----------- resource/utilities/resource-query.cpp | 14 ++++++++++++-- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/resource/utilities/command.hpp b/resource/utilities/command.hpp index b365d5469..d9b7390b2 100644 --- a/resource/utilities/command.hpp +++ b/resource/utilities/command.hpp @@ -25,17 +25,18 @@ namespace Flux { namespace resource_model { struct test_params_t { - std::string load_file; /* load file name */ - std::string load_format; /* load reader format */ - std::string load_allowlist; /* load resource allowlist */ - std::string matcher_name; /* Matcher name */ - std::string matcher_policy; /* Matcher policy name */ - std::string o_fname; /* Output file to dump the filtered graph */ - std::ofstream r_out; /* Output file stream for emitted R */ - std::string r_fname; /* Output file to dump the emitted R */ - std::string o_fext; /* File extension */ - std::string prune_filters; /* Raw prune-filter specification */ - std::string match_format; /* Format to emit a matched resources */ + std::string load_file; /* load file name */ + std::string load_format; /* load reader format */ + std::string load_allowlist; /* load resource allowlist */ + std::string matcher_name; /* Matcher name */ + std::string matcher_policy; /* Matcher policy name */ + std::string traverser_policy; /* Traverser policy name */ + std::string o_fname; /* Output file to dump the filtered graph */ + std::ofstream r_out; /* Output file stream for emitted R */ + std::string r_fname; /* Output file to dump the emitted R */ + std::string o_fext; /* File extension */ + std::string prune_filters; /* Raw prune-filter specification */ + std::string match_format; /* Format to emit a matched resources */ emit_format_t o_format; bool elapse_time; /* Print elapse time */ bool disable_prompt; /* Disable resource-query> prompt */ diff --git a/resource/utilities/resource-query.cpp b/resource/utilities/resource-query.cpp index 6d6b6dff7..a29bb47aa 100644 --- a/resource/utilities/resource-query.cpp +++ b/resource/utilities/resource-query.cpp @@ -32,12 +32,13 @@ extern "C" { #include "resource/utilities/command.hpp" #include "resource/store/resource_graph_store.hpp" #include "resource/policies/dfu_match_policy_factory.hpp" +#include "resource/traversers/dfu_traverser_policy_factory.hpp" namespace fs = std::filesystem; using namespace Flux::resource_model; using boost::tie; -#define OPTIONS "L:f:W:S:P:F:g:o:p:t:r:edh" +#define OPTIONS "L:f:W:S:P:T:F:g:o:p:t:r:edh" static const struct option longopts[] = { {"load-file", required_argument, 0, 'L'}, {"load-format", required_argument, 0, 'f'}, @@ -45,6 +46,7 @@ static const struct option longopts[] = { {"match-subsystems", required_argument, 0, 'S'}, {"match-policy", required_argument, 0, 'P'}, {"match-format", required_argument, 0, 'F'}, + {"traverser-policy", required_argument, 0, 'T'}, {"graph-format", required_argument, 0, 'g'}, {"graph-output", required_argument, 0, 'o'}, {"prune-filters", required_argument, 0, 'p'}, @@ -147,6 +149,10 @@ To see cli commands, type in "help" in the cli: i.e., Specify the emit format of the matched resource set. (default=simple). + -T, --traverser-policy= + Specify the traverser policy. + (default=simple) + -p, --prune-filters= Install a planner-based filter at each High-Level (HL) resource vertex which tracks the state of the Low-Level (LL) resources @@ -239,6 +245,7 @@ static void set_default_params (std::shared_ptr &ctx) ctx->params.load_allowlist = ""; ctx->params.matcher_name = "CA"; ctx->params.matcher_policy = "first"; + ctx->params.traverser_policy = "simple"; ctx->params.o_fname = ""; ctx->params.r_fname = ""; ctx->params.o_fext = "dot"; @@ -570,7 +577,7 @@ static int init_resource_graph (std::shared_ptr &ctx) } try { - ctx->traverser = std::make_shared (); + ctx->traverser = create_traverser (ctx->params.traverser_policy); } catch (std::bad_alloc &e) { errno = ENOMEM; std::cerr << "ERROR: out of memory allocating traverser" << std::endl; @@ -637,6 +644,9 @@ static void process_args (std::shared_ptr &ctx, int argc, ch case 'P': /* --match-policy */ ctx->params.matcher_policy = optarg; break; + case 'T': /* --traverser-policy */ + ctx->params.traverser_policy = optarg; + break; case 'F': /* --match-format */ ctx->params.match_format = optarg; if (!known_match_format (ctx->params.match_format)) { From 9a0ee5c71eb851c8a03cc393e1dc30135cff9b09 Mon Sep 17 00:00:00 2001 From: Zeke Morton Date: Wed, 29 Jan 2025 16:41:28 -0800 Subject: [PATCH 8/8] test: update tests for changes in flexible scheduling Problem: new changes in flexible scheduling now require a new command line option to enable the feature in resource query and no longer use or_slot but just slot Add in trverser policy option and update jobspecs to use slot instead of or_slot --- t/data/resource/jobspecs/flexible/test001.yaml | 4 ++-- t/data/resource/jobspecs/flexible/test002.yaml | 4 ++-- t/data/resource/jobspecs/flexible/test003.yaml | 6 +++--- t/data/resource/jobspecs/flexible/test004.yaml | 2 +- t/data/resource/jobspecs/flexible/test005.yaml | 4 ++-- t/data/resource/jobspecs/flexible/test006.yaml | 2 +- t/data/resource/jobspecs/flexible/test007.yaml | 2 +- t/data/resource/jobspecs/flexible/test008.yaml | 4 ++-- t/t3037-resource-flexible.t | 16 ++++++++-------- 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/t/data/resource/jobspecs/flexible/test001.yaml b/t/data/resource/jobspecs/flexible/test001.yaml index cbfaf5bcb..fd7c8c4b3 100644 --- a/t/data/resource/jobspecs/flexible/test001.yaml +++ b/t/data/resource/jobspecs/flexible/test001.yaml @@ -12,7 +12,7 @@ resources: - type: socket count: 1 with: - - type: or_slot + - type: slot count: 1 label: small with: @@ -20,7 +20,7 @@ resources: count: 8 - type: gpu count: 1 - - type: or_slot + - type: slot count: 1 label: big with: diff --git a/t/data/resource/jobspecs/flexible/test002.yaml b/t/data/resource/jobspecs/flexible/test002.yaml index 2f627b5f9..ca6713815 100644 --- a/t/data/resource/jobspecs/flexible/test002.yaml +++ b/t/data/resource/jobspecs/flexible/test002.yaml @@ -12,7 +12,7 @@ resources: - type: socket count: 1 with: - - type: or_slot + - type: slot count: 1 label: small with: @@ -20,7 +20,7 @@ resources: count: 8 - type: gpu count: 1 - - type: or_slot + - type: slot count: 1 label: big with: diff --git a/t/data/resource/jobspecs/flexible/test003.yaml b/t/data/resource/jobspecs/flexible/test003.yaml index 604e9967b..aba01053d 100644 --- a/t/data/resource/jobspecs/flexible/test003.yaml +++ b/t/data/resource/jobspecs/flexible/test003.yaml @@ -1,6 +1,6 @@ version: 9999 resources: - - type: or_slot + - type: slot count: 2 label: default with: @@ -8,7 +8,7 @@ resources: count: 12 - type: memory count: 4 - - type: or_slot + - type: slot count: 2 label: default with: @@ -18,7 +18,7 @@ resources: count: 1 - type: memory count: 4 - - type: or_slot + - type: slot count: 2 label: default with: diff --git a/t/data/resource/jobspecs/flexible/test004.yaml b/t/data/resource/jobspecs/flexible/test004.yaml index b921f3115..cf0db5e18 100644 --- a/t/data/resource/jobspecs/flexible/test004.yaml +++ b/t/data/resource/jobspecs/flexible/test004.yaml @@ -8,7 +8,7 @@ resources: count: 8 - type: memory count: 2 - - type: or_slot + - type: slot count: 1 label: default with: diff --git a/t/data/resource/jobspecs/flexible/test005.yaml b/t/data/resource/jobspecs/flexible/test005.yaml index 60ebd66c5..468b1b3cc 100644 --- a/t/data/resource/jobspecs/flexible/test005.yaml +++ b/t/data/resource/jobspecs/flexible/test005.yaml @@ -1,6 +1,6 @@ version: 9999 resources: - - type: or_slot + - type: slot count: 2 label: default with: @@ -16,7 +16,7 @@ resources: count: 1 - type: memory count: 6 - - type: or_slot + - type: slot count: 2 label: default with: diff --git a/t/data/resource/jobspecs/flexible/test006.yaml b/t/data/resource/jobspecs/flexible/test006.yaml index d0eb498e1..f436ac53f 100644 --- a/t/data/resource/jobspecs/flexible/test006.yaml +++ b/t/data/resource/jobspecs/flexible/test006.yaml @@ -1,6 +1,6 @@ version: 9999 resources: - - type: or_slot + - type: slot count: 1 label: default with: diff --git a/t/data/resource/jobspecs/flexible/test007.yaml b/t/data/resource/jobspecs/flexible/test007.yaml index 40521a71f..fc7680096 100644 --- a/t/data/resource/jobspecs/flexible/test007.yaml +++ b/t/data/resource/jobspecs/flexible/test007.yaml @@ -7,7 +7,7 @@ resources: count: 8 - type: memory count: 2 - - type: or_slot + - type: slot count: 1 label: default with: diff --git a/t/data/resource/jobspecs/flexible/test008.yaml b/t/data/resource/jobspecs/flexible/test008.yaml index 730132bf8..841fce6bd 100644 --- a/t/data/resource/jobspecs/flexible/test008.yaml +++ b/t/data/resource/jobspecs/flexible/test008.yaml @@ -1,12 +1,12 @@ version: 9999 resources: - - type: or_slot + - type: slot count: 1 label: default with: - type: core count: 1 - - type: or_slot + - type: slot count: 1 label: default with: diff --git a/t/t3037-resource-flexible.t b/t/t3037-resource-flexible.t index f4281d975..e7551085d 100755 --- a/t/t3037-resource-flexible.t +++ b/t/t3037-resource-flexible.t @@ -13,7 +13,7 @@ cmds001="${cmd_dir}/cmds01.in" test001_desc="JGF: allocate 9 jobspecs with flexible scheduling" test_expect_success "${test001_desc}" ' sed "s~@TEST_SRCDIR@~${SHARNESS_TEST_SRCDIR}~g" ${cmds001} > cmds001 && - ${query} -L ${jgf} -f jgf -S CA -P high -t 001.R.out < cmds001 && + ${query} -L ${jgf} -f jgf -S CA -P high -T flexible -t 001.R.out < cmds001 && test_cmp 001.R.out ${exp_dir}/001.R.out ' @@ -21,7 +21,7 @@ cmds002="${cmd_dir}/cmds02.in" test002_desc="JGF: allocate jobspecs with valid and invalid or_slots" test_expect_success "${test002_desc}" ' sed "s~@TEST_SRCDIR@~${SHARNESS_TEST_SRCDIR}~g" ${cmds002} > cmds002 && - ${query} -L ${jgf} -f jgf -S CA -P high -t 002.R.out < cmds002 && + ${query} -L ${jgf} -f jgf -S CA -P high -T flexible -t 002.R.out < cmds002 && test_cmp 002.R.out ${exp_dir}/002.R.out ' @@ -29,7 +29,7 @@ cmds003="${cmd_dir}/cmds03.in" test003_desc="JGF: allocate jobspecs with pristine jobspec" test_expect_success "${test003_desc}" ' sed "s~@TEST_SRCDIR@~${SHARNESS_TEST_SRCDIR}~g" ${cmds003} > cmds003 && - ${query} -L ${jgf} -f jgf -S CA -P high -t 003.R.out < cmds003 && + ${query} -L ${jgf} -f jgf -S CA -P high -T flexible -t 003.R.out < cmds003 && test_cmp 003.R.out ${exp_dir}/003.R.out ' @@ -37,7 +37,7 @@ cmds004="${cmd_dir}/cmds04.in" test004_desc="JGF: try to allocate jobspec with slot and or_slot siblings" test_expect_success "${test004_desc}" ' sed "s~@TEST_SRCDIR@~${SHARNESS_TEST_SRCDIR}~g" ${cmds004} > cmds004 && - ${query} -L ${jgf} -f jgf -S CA -P high -t 004.R.out < cmds004 && + ${query} -L ${jgf} -f jgf -S CA -P high -T flexible -t 004.R.out < cmds004 && test_cmp 004.R.out ${exp_dir}/004.R.out ' @@ -45,7 +45,7 @@ cmds005="${cmd_dir}/cmds05.in" test005_desc="JGF: try to allocate jobspec with or_slot count > 1" test_expect_success "${test005_desc}" ' sed "s~@TEST_SRCDIR@~${SHARNESS_TEST_SRCDIR}~g" ${cmds005} > cmds005 && - ${query} -L ${jgf} -f jgf -S CA -P high -t 005.R.out < cmds005 && + ${query} -L ${jgf} -f jgf -S CA -P high -T flexible -t 005.R.out < cmds005 && test_cmp 005.R.out ${exp_dir}/005.R.out ' @@ -53,7 +53,7 @@ cmds006="${cmd_dir}/cmds06.in" test006_desc="JGF: try to allocate pristine jobspec with resources that don't exist" test_expect_success "${test006_desc}" ' sed "s~@TEST_SRCDIR@~${SHARNESS_TEST_SRCDIR}~g" ${cmds006} > cmds006 && - ${query} -L ${jgf} -f jgf -S CA -P high -t 006.R.out < cmds006 && + ${query} -L ${jgf} -f jgf -S CA -P high -T flexible -t 006.R.out < cmds006 && test_cmp 006.R.out ${exp_dir}/006.R.out ' @@ -61,7 +61,7 @@ cmds007="${cmd_dir}/cmds07.in" test007_desc="JGF: try to allocate jobspec with resource and or_slot siblings" test_expect_success "${test007_desc}" ' sed "s~@TEST_SRCDIR@~${SHARNESS_TEST_SRCDIR}~g" ${cmds007} > cmds007 && - ${query} -L ${jgf} -f jgf -S CA -P high -t 007.R.out < cmds007 && + ${query} -L ${jgf} -f jgf -S CA -P high -T flexible -t 007.R.out < cmds007 && test_cmp 007.R.out ${exp_dir}/007.R.out ' @@ -69,7 +69,7 @@ cmds008="${cmd_dir}/cmds08.in" test008_desc="JGF: try to allocate jobspec with identical or_slot configurations" test_expect_success "${test008_desc}" ' sed "s~@TEST_SRCDIR@~${SHARNESS_TEST_SRCDIR}~g" ${cmds008} > cmds008 && - ${query} -L ${jgf} -f jgf -S CA -P high -t 008.R.out < cmds008 && + ${query} -L ${jgf} -f jgf -S CA -P high -T flexible -t 008.R.out < cmds008 && test_cmp 008.R.out ${exp_dir}/008.R.out '