Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

hopefully fix ci issue #16773 #34

Merged
merged 9 commits into from
Apr 8, 2021
+116 −107
Merged

hopefully fix ci issue #16773 #34

Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
bbc804f
hopefully fix ci issue #16773
strega-nil Mar 18, 2021
34619df
format & fix build
strega-nil Mar 19, 2021
98f640b
ooh, new clang-format version
strega-nil Mar 30, 2021
da67049
add host exclusions
strega-nil Mar 31, 2021
84b6c0c
format!
strega-nil Mar 31, 2021
811f74e
oops
strega-nil Apr 1, 2021
2a2e96a
format with the right clang-format
strega-nil Apr 1, 2021
5405698
Apply suggestions from code review
strega-nil Apr 7, 2021
48f8b3c
format
strega-nil Apr 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
223 changes: 116 additions & 107 deletions src/vcpkg/commands.ci.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,12 +88,14 @@ namespace vcpkg::Commands::CI

static constexpr StringLiteral OPTION_DRY_RUN = "dry-run";
static constexpr StringLiteral OPTION_EXCLUDE = "exclude";
static constexpr StringLiteral OPTION_HOST_EXCLUDE = "host-exclude";
static constexpr StringLiteral OPTION_FAILURE_LOGS = "failure-logs";
static constexpr StringLiteral OPTION_XUNIT = "x-xunit";
static constexpr StringLiteral OPTION_RANDOMIZE = "x-randomize";

static constexpr std::array<CommandSetting, 3> CI_SETTINGS = {
static constexpr std::array<CommandSetting, 4> CI_SETTINGS = {
{{OPTION_EXCLUDE, "Comma separated list of ports to skip"},
{OPTION_HOST_EXCLUDE, "Comma separated list of ports to skip for the host triplet"},
{OPTION_XUNIT, "File to output results in XUnit format (internal)"},
{OPTION_FAILURE_LOGS, "Directory to which failure logs will be copied"}}};

Expand All @@ -105,7 +107,7 @@ namespace vcpkg::Commands::CI
const CommandStructure COMMAND_STRUCTURE = {
create_example_string("ci x64-windows"),
1,
SIZE_MAX,
1,
{CI_SWITCHES, CI_SETTINGS},
nullptr,
};
Expand Down Expand Up @@ -291,6 +293,7 @@ namespace vcpkg::Commands::CI
static std::unique_ptr<UnknownCIPortsResults> find_unknown_ports_for_ci(
const VcpkgPaths& paths,
const std::set<std::string>& exclusions,
const std::set<std::string>& host_exclusions,
const PortFileProvider::PortFileProvider& provider,
const CMakeVars::CMakeVarProvider& var_provider,
const std::vector<FullPackageSpec>& specs,
Expand All @@ -300,6 +303,17 @@ namespace vcpkg::Commands::CI
{
auto ret = std::make_unique<UnknownCIPortsResults>();

auto is_excluded = [&](const PackageSpec& spec) -> bool {
if (spec.triplet() == host_triplet)
{
return Util::Sets::contains(host_exclusions, spec.name());
}
else
{
return Util::Sets::contains(exclusions, spec.name());
}
};

std::set<PackageSpec> will_fail;

std::vector<PackageSpec> packages_with_qualified_deps;
Expand Down Expand Up @@ -340,6 +354,8 @@ namespace vcpkg::Commands::CI

for (auto&& action : action_plan.install_actions)
{
action.build_options = vcpkg::Build::backcompat_prohibiting_package_options;

auto p = &action;
ret->abi_map.emplace(action.spec, action.abi_info.value_or_exit(VCPKG_LINE_INFO).package_abi);
ret->features.emplace(action.spec, action.feature_list);
Expand All @@ -349,11 +365,12 @@ namespace vcpkg::Commands::CI

std::string state;

if (Util::Sets::contains(exclusions, p->spec.name()))
if (is_excluded(p->spec))
{
state = "skip";
ret->known.emplace(p->spec, BuildResult::EXCLUDED);
will_fail.emplace(p->spec);
action.plan_type = InstallPlanType::EXCLUDED;
}
else if (!supported_for_triplet(var_provider, p))
{
Expand Down Expand Up @@ -424,10 +441,21 @@ namespace vcpkg::Commands::CI
return ret;
}

void perform_and_exit(const VcpkgCmdArguments& args,
const VcpkgPaths& paths,
Triplet default_triplet,
Triplet host_triplet)
static std::set<std::string> parse_exclusions(const ParsedArguments& options, StringLiteral opt)
{
std::set<std::string> exclusions_set;
auto it_exclusions = options.settings.find(opt);
if (it_exclusions != options.settings.end())
{
auto exclusions = Strings::split(it_exclusions->second, ',');
exclusions_set.insert(std::make_move_iterator(exclusions.begin()),
std::make_move_iterator(exclusions.end()));
}

return exclusions_set;
}

void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths, Triplet, Triplet host_triplet)
{
std::unique_ptr<IBinaryProvider> binaryproviderStorage;
if (args.binary_caching_enabled())
Expand All @@ -438,26 +466,19 @@ namespace vcpkg::Commands::CI

IBinaryProvider& binaryprovider = binaryproviderStorage ? *binaryproviderStorage : null_binary_provider();

const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE);
auto& settings = options.settings;

std::set<std::string> exclusions_set;
auto it_exclusions = settings.find(OPTION_EXCLUDE);
if (it_exclusions != settings.end())
if (args.command_arguments.size() != 1)
{
auto exclusions = Strings::split(it_exclusions->second, ',');
exclusions_set.insert(exclusions.begin(), exclusions.end());
Checks::unreachable(VCPKG_LINE_INFO);
}

const auto is_dry_run = Util::Sets::contains(options.switches, OPTION_DRY_RUN);
const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE);
const auto& settings = options.settings;

std::vector<Triplet> triplets = Util::fmap(
args.command_arguments, [](std::string s) { return Triplet::from_canonical_name(std::move(s)); });
Triplet target_triplet = Triplet::from_canonical_name(std::string(args.command_arguments[0]));
auto exclusions_set = parse_exclusions(options, OPTION_EXCLUDE);
auto host_exclusions_set = parse_exclusions(options, OPTION_HOST_EXCLUDE);

if (triplets.empty())
{
triplets.push_back(default_triplet);
}
const auto is_dry_run = Util::Sets::contains(options.switches, OPTION_DRY_RUN);

auto& filesystem = paths.get_filesystem();
Optional<CiBuildLogsRecorder> build_logs_recorder_storage;
Expand Down Expand Up @@ -489,110 +510,98 @@ namespace vcpkg::Commands::CI
Util::fmap(provider.load_all_control_files(), Paragraphs::get_name_of_control_file);
std::vector<TripletAndSummary> results;
auto timer = Chrono::ElapsedTimer::create_started();
for (Triplet triplet : triplets)
{
Input::check_triplet(triplet, paths);

xunitTestResults.push_collection(triplet.canonical_name());
Input::check_triplet(target_triplet, paths);

std::vector<PackageSpec> specs = PackageSpec::to_package_specs(all_ports, triplet);
// Install the default features for every package
auto all_default_full_specs = Util::fmap(specs, [&](auto& spec) {
std::vector<std::string> default_features =
provider.get_control_file(spec.name()).get()->source_control_file->core_paragraph->default_features;
default_features.emplace_back("core");
return FullPackageSpec{spec, std::move(default_features)};
});
xunitTestResults.push_collection(target_triplet.canonical_name());

Dependencies::CreateInstallPlanOptions serialize_options(host_triplet);
std::vector<PackageSpec> specs = PackageSpec::to_package_specs(all_ports, target_triplet);
// Install the default features for every package
auto all_default_full_specs = Util::fmap(specs, [&](auto& spec) {
std::vector<std::string> default_features =
provider.get_control_file(spec.name()).get()->source_control_file->core_paragraph->default_features;
default_features.emplace_back("core");
return FullPackageSpec{spec, std::move(default_features)};
});

struct RandomizerInstance : Graphs::Randomizer
{
virtual int random(int i) override
{
if (i <= 1) return 0;
std::uniform_int_distribution<int> d(0, i - 1);
return d(e);
}

std::random_device e;
} randomizer_instance;
Dependencies::CreateInstallPlanOptions serialize_options(host_triplet);

if (Util::Sets::contains(options.switches, OPTION_RANDOMIZE))
struct RandomizerInstance : Graphs::Randomizer
{
virtual int random(int i) override
{
serialize_options.randomizer = &randomizer_instance;
if (i <= 1) return 0;
std::uniform_int_distribution<int> d(0, i - 1);
return d(e);
}

auto split_specs = find_unknown_ports_for_ci(paths,
exclusions_set,
provider,
var_provider,
all_default_full_specs,
binaryprovider,
serialize_options,
host_triplet);
std::random_device e;
} randomizer_instance;

if (Util::Sets::contains(options.switches, OPTION_RANDOMIZE))
{
serialize_options.randomizer = &randomizer_instance;
}

auto split_specs = find_unknown_ports_for_ci(paths,
exclusions_set,
host_exclusions_set,
provider,
var_provider,
all_default_full_specs,
binaryprovider,
serialize_options,
host_triplet);

auto& action_plan = split_specs->plan;
auto& action_plan = split_specs->plan;

for (auto&& action : action_plan.install_actions)
if (is_dry_run)
{
Dependencies::print_plan(action_plan, true, paths.builtin_ports_directory());
}
else
{
auto collection_timer = Chrono::ElapsedTimer::create_started();
auto summary = Install::perform(args,
action_plan,
Install::KeepGoing::YES,
paths,
status_db,
binaryprovider,
build_logs_recorder,
var_provider);
auto collection_time_elapsed = collection_timer.elapsed();

// Adding results for ports that were built or pulled from an archive
for (auto&& result : summary.results)
{
if (Util::Sets::contains(exclusions_set, action.spec.name()))
{
action.plan_type = InstallPlanType::EXCLUDED;
}
else
{
action.build_options = vcpkg::Build::backcompat_prohibiting_package_options;
}
auto& port_features = split_specs->features.at(result.spec);
split_specs->known.erase(result.spec);
xunitTestResults.add_test_results(result.spec.to_string(),
result.build_result.code,
result.timing,
split_specs->abi_map.at(result.spec),
port_features);
}

if (is_dry_run)
// Adding results for ports that were not built because they have known states
for (auto&& port : split_specs->known)
{
Dependencies::print_plan(action_plan, true, paths.builtin_ports_directory());
auto& port_features = split_specs->features.at(port.first);
xunitTestResults.add_test_results(port.first.to_string(),
port.second,
Chrono::ElapsedTime{},
split_specs->abi_map.at(port.first),
port_features);
}
else
{
auto collection_timer = Chrono::ElapsedTimer::create_started();
auto summary = Install::perform(args,
action_plan,
Install::KeepGoing::YES,
paths,
status_db,
binaryprovider,
build_logs_recorder,
var_provider);
auto collection_time_elapsed = collection_timer.elapsed();

// Adding results for ports that were built or pulled from an archive
for (auto&& result : summary.results)
{
auto& port_features = split_specs->features.at(result.spec);
split_specs->known.erase(result.spec);
xunitTestResults.add_test_results(result.spec.to_string(),
result.build_result.code,
result.timing,
split_specs->abi_map.at(result.spec),
port_features);
}

// Adding results for ports that were not built because they have known states
for (auto&& port : split_specs->known)
{
auto& port_features = split_specs->features.at(port.first);
xunitTestResults.add_test_results(port.first.to_string(),
port.second,
Chrono::ElapsedTime{},
split_specs->abi_map.at(port.first),
port_features);
}
all_known_results.emplace_back(std::move(split_specs->known));

all_known_results.emplace_back(std::move(split_specs->known));
results.push_back({target_triplet, std::move(summary)});

results.push_back({triplet, std::move(summary)});

xunitTestResults.collection_time(collection_time_elapsed);
}
xunitTestResults.collection_time(collection_time_elapsed);
}

xunitTestResults.assembly_time(timer.elapsed());

for (auto&& result : results)
Expand Down