Merge branch 'main' into 4534

.buildkite/common.py

@@ -13,19 +13,18 @@
 import subprocess
 from pathlib import Path
 
-DEFAULT_INSTANCES = [
- "c5n.metal", # Intel Skylake
- "m5n.metal", # Intel Cascade Lake
- "m6i.metal", # Intel Icelake
- "m6a.metal", # AMD Milan
- "m6g.metal", # Graviton2
- "m7g.metal", # Graviton3
-]
+DEFAULT_INSTANCES = {
+ "c5n.metal": "x86_64", # Intel Skylake
+ "m5n.metal": "x86_64", # Intel Cascade Lake
+ "m6i.metal": "x86_64", # Intel Icelake
+ "m6a.metal": "x86_64", # AMD Milan
+ "m6g.metal": "aarch64", # Graviton2
+ "m7g.metal": "aarch64", # Graviton3
+}
 
 DEFAULT_PLATFORMS = [
  ("al2", "linux_5.10"),
- # TODO: unpin 6.1 AMI once the bug is fixed
- ("al2023", "linux_6.1-pinned"),
+ ("al2023", "linux_6.1"),
 ]
 
 
@@ -146,7 +145,7 @@ def __call__(self, parser, namespace, value, option_string=None):
  "--instances",
  required=False,
  nargs="+",
- default=DEFAULT_INSTANCES,
+ default=DEFAULT_INSTANCES.keys(),
 )
 COMMON_PARSER.add_argument(
  "--platforms",
@@ -233,7 +232,7 @@ class BKPipeline:
 
  parser = COMMON_PARSER
 
- def __init__(self, initial_steps=None, with_build_step=True, **kwargs):
+ def __init__(self, with_build_step=True, **kwargs):
  self.steps = []
  self.args = args = self.parser.parse_args()
  # Retry one time if agent was lost. This can happen if we terminate the
@@ -258,33 +257,22 @@ def __init__(self, initial_steps=None, with_build_step=True, **kwargs):
  # Build sharing
  if with_build_step:
  build_cmds, self.shared_build = shared_build()
- step_build = group("🏗️ Build", build_cmds, **self.per_arch)
- self.steps += [step_build, "wait"]
+ self.build_group_per_arch(
+ "🏗️ Build", build_cmds, depends_on_build=False, set_key=True
+ )
  else:
  self.shared_build = None
 
- # If we run initial_steps before the "wait" step above, then a failure of the initial steps
- # would result in the build not progressing past the "wait" step (as buildkite only proceeds past a wait step
- # if everything before it passed). Thus put the initial steps after the "wait" step, but set `"depends_on": null`
- # to start running them immediately (e.g. without waiting for the "wait" step to unblock).
- #
- # See also https://buildkite.com/docs/pipelines/dependencies#explicit-dependencies-in-uploaded-steps
- if initial_steps:
- for step in initial_steps:
- step["depends_on"] = None
-
- self.steps += initial_steps
-
- def add_step(self, step, decorate=True):
+ def add_step(self, step, depends_on_build=True):
  """
  Add a step to the pipeline.
 
  https://buildkite.com/docs/pipelines/step-reference
 
  :param step: a Buildkite step
- :param decorate: inject needed commands for sharing builds
+ :param depends_on_build: inject needed commands for sharing builds
  """
- if decorate and isinstance(step, dict):
+ if depends_on_build and isinstance(step, dict):
  step = self._adapt_group(step)
  self.steps.append(step)
  return step
@@ -307,6 +295,10 @@ def _adapt_group(self, group):
 
  for step in group["steps"]:
  step["command"] = prepend + step["command"]
+ if self.shared_build is not None:
+ step["depends_on"] = self.build_key(
+ DEFAULT_INSTANCES[step["agents"]["instance"]]
+ )
  return group
 
  def build_group(self, *args, **kwargs):
@@ -315,16 +307,34 @@ def build_group(self, *args, **kwargs):
 
  https://buildkite.com/docs/pipelines/group-step
  """
- decorate = kwargs.pop("decorate", True)
+ depends_on_build = kwargs.pop("depends_on_build", True)
  combined = overlay_dict(self.per_instance, kwargs)
- return self.add_step(group(*args, **combined), decorate=decorate)
+ return self.add_step(
+ group(*args, **combined), depends_on_build=depends_on_build
+ )
+
+ def build_key(self, arch):
+ """Return the Buildkite key for the build step, for the specified arch"""
+ return self.shared_build.replace("$(uname -m)", arch).replace(".tar.gz", "")
 
- def build_group_per_arch(self, *args, **kwargs):
+ def build_group_per_arch(self, label, *args, **kwargs):
  """
  Build a group, parametrizing over the architectures only.
+
+ kwargs consumed by this method and not passed down to `group`:
+ - `depends_on_build` (default: `True`): Whether the steps in this group depend on the artifacts from the shared compilation steps
+ - `set_key`: If True, causes the generated steps to have a "key" field
  """
+ depends_on_build = kwargs.pop("depends_on_build", True)
+ set_key = kwargs.pop("set_key", None)
  combined = overlay_dict(self.per_arch, kwargs)
- return self.add_step(group(*args, **combined))
+ grp = group(label, *args, **combined)
+ if set_key:
+ for step in grp["steps"]:
+ step["key"] = self.build_key(
+ DEFAULT_INSTANCES[step["agents"]["instance"]]
+ )
+ return self.add_step(grp, depends_on_build=depends_on_build)
 
  def to_dict(self):
  """Render the pipeline as a dictionary."""

.buildkite/pipeline_cpu_template.py

@@ -34,7 +34,7 @@ class BkStep(str, Enum):
  "tools/devtool -y test --no-build -- -m no_block_pr integration_tests/functional/test_cpu_template_helper.py -k test_guest_cpu_config_change",
  ],
  BkStep.LABEL: "🖐️ fingerprint",
- "instances": DEFAULT_INSTANCES,
+ "instances": DEFAULT_INSTANCES.keys(),
  "platforms": DEFAULT_PLATFORMS,
  },
  "cpuid_wrmsr": {
@@ -117,11 +117,11 @@ def group_snapshot_restore(test_step):
  BkStep.COMMAND: restore_commands,
  BkStep.LABEL: restore_label,
  BkStep.TIMEOUT: test_step["restore"][BkStep.TIMEOUT],
- "agents": [
- f"instance={restore_instance}",
- f"kv={restore_kv}",
- f"os={restore_os}",
- ],
+ "agents": {
+ "instance": restore_instance,
+ "kv": restore_kv,
+ "os": restore_os,
+ },
  }
  )
 

.buildkite/pipeline_pr.py

@@ -21,15 +21,17 @@
 pipeline = BKPipeline(
  priority=DEFAULT_PRIORITY,
  timeout_in_minutes=45,
- initial_steps=[
- {
- "command": "./tools/devtool -y checkstyle",
- "label": "🪶 Style",
- },
- ],
  with_build_step=not DOC_ONLY_CHANGE,
 )
 
+pipeline.add_step(
+ {
+ "command": "./tools/devtool -y checkstyle",
+ "label": "🪶 Style",
+ },
+ depends_on_build=False,
+)
+
 # run sanity build of devtool if Dockerfile is changed
 if any(x.parent.name == "devctr" for x in changed_files):
  pipeline.build_group_per_arch(
@@ -58,17 +60,17 @@
  platforms=[("al2", "linux_5.10")],
  timeout_in_minutes=300,
  **DEFAULTS_PERF,
- decorate=False,
+ depends_on_build=False,
  )
  # modify Kani steps' label
  for step in kani_grp["steps"]:
  step["label"] = "🔍 Kani"
- kani_grp["depends_on"] = None
 
 if run_all_tests(changed_files):
  pipeline.build_group(
  "📦 Build",
  pipeline.devtool_test(pytest_opts="integration_tests/build/"),
+ depends_on_build=False,
  )
 
  pipeline.build_group(

CHANGELOG.md

@@ -16,12 +16,20 @@ and this project adheres to
 
 ### Removed
 
+- [#4804](https://github.com/firecracker-microvm/firecracker/pull/4804): Drop
+ Support for guest kernel 4.14. Linux 4.14 reached end-of-life in
+ [January 2024](https://lore.kernel.org/lkml/2024011046-ecology-tiptoeing-ce50@gregkh/)
+ The minimum supported guest kernel now is 5.10.
+
 ### Fixed
 
 - [#4796](https://github.com/firecracker-microvm/firecracker/pull/4796): Fixed
  Vsock not notifying guest about `TRANSPORT_RESET_EVENT` event after snapshot
  restore. This resulted in guest waiting indefinitely on a connection which was
  reset during snapshot creation.
+- [#4790](https://github.com/firecracker-microvm/firecracker/pull/4790): v1.9.0
+ was missing most of the debugging information in the debuginfo file, due to a
+ change in the Cargo defaults. This has been corrected.
 
 ## \[1.9.0\]
 

Cargo.toml

@@ -27,3 +27,7 @@ panic = "abort"
 [profile.release]
 panic = "abort"
 lto = true
+strip = "none"
+
+[profile.bench]
+strip = "debuginfo"