Add unmanaged nodes integration tests (#2780)

* Add docker file to the runtime tools * fixes * bug fixes * more bug fixes and added doc * don;t overwrite the RUST_LOG env var * integration test for unmanaged nodes * add unamanged parameters to launch() * add ing object_id * more bug fixes * bug fixes * chmod on the linux files in docker * format * cleanup merge * added test_unmanaged command * cleanup * use a single image for the docker compose remove images after the test * docs and formatting * format * format * format and bug fixes * using windows server * fix linux container make the base image a paramter on windows use the windows server base image on windows server * format * bug fix * more fixes * allow reboot * more fixes * added more logging around the service principal creation * format * more logging * change restart policy * fix multi tenant domain * more fixes * exit instead of reboot when running inside docker * remove comment * build fix * try_exist instead of exist * save the docker logs * bug_fix * adding timeout * fix timeout logic * adding a build profile * make all agent depend on the first one * remove profile * another fix * restart agent 1 * Update docs/unmnaged-nodes.md Co-authored-by: Teo Voinea <58236992+tevoinea@users.noreply.github.com> --------- Co-authored-by: Teo Voinea <58236992+tevoinea@users.noreply.github.com>
microsoft · Feb 8, 2023 · d732028 · d732028
1 parent f93c755
commit d732028
Show file tree

Hide file tree

Showing 9 changed files with 404 additions and 51 deletions.
diff --git a/docs/unmnaged-nodes.md b/docs/unmnaged-nodes.md
@@ -87,4 +87,34 @@ onefuzz nodes get <machine_guid>
 ```
 
 This should return one entry. Verify that the `pool_name` matched the pool name created earlier.
-From here you will be able to schedule jobs on that pool and they will be running.
+From here you will be able to schedule jobs on that pool and they will be running.
+
+
+## Troubleshooting
+
+### increase the verbosity of the logs
+It can help when investigating issues to increase the log verbosity. you will need to set the [RUST_LOG](https://docs.rs/env_logger/latest/env_logger/#enabling-logging) environment variable when starting docker
+
+```
+docker run --rm --env RUST_LOG=<log_level> <image_name> --machine_id <machine_id>
+```
+log_level can be any of
+- error
+- warn
+- info
+- debug
+- trace
+
+
+### use the container interactively
+you can use the container interactively by with the following command
+
+windows
+```
+docker run --it --rm --entrypoint powershell <image_name>
+```
+
+linux
+```
+docker run --it --rm --entrypoint bash <image_name>
+```
diff --git a/src/ApiService/ApiService/onefuzzlib/Config.cs b/src/ApiService/ApiService/onefuzzlib/Config.cs
@@ -320,10 +320,6 @@ public async Async.Task<ResultVoid<TaskConfigError>> CheckConfig(TaskConfig conf
             return ResultVoid<TaskConfigError>.Error(new TaskConfigError($"invalid pool: {config.Pool.PoolName}"));
         }
 
-        if ((config.Task.RebootAfterSetup ?? false) && !pool.OkV.Managed) {
-            return ResultVoid<TaskConfigError>.Error(new TaskConfigError("reboot_after_setup is not supported for unmanaged pools"));
-        }
-
         var checkTarget = await CheckTargetExe(config, definition);
         if (!checkTarget.IsOk) {
             return checkTarget;

diff --git a/src/agent/onefuzz-agent/src/reboot.rs b/src/agent/onefuzz-agent/src/reboot.rs
@@ -90,17 +90,22 @@ impl Reboot {
 
     #[cfg(target_family = "unix")]
     pub fn invoke(&self) -> Result<()> {
-        info!("invoking local reboot command");
-
-        Command::new("reboot").arg("-f").status()?;
-
-        self.wait_for_reboot()
+        match std::path::Path::new("/.dockerenv").try_exists() {
+            Ok(true) => {
+                info!("running inside docker, exiting instead of rebooting");
+                std::process::exit(0);
+            }
+            _ => {
+                info!("invoking local reboot command");
+                Command::new("reboot").arg("-f").status()?;
+                self.wait_for_reboot()
+            }
+        }
     }
 
     #[cfg(target_family = "windows")]
     pub fn invoke(&self) -> Result<()> {
         info!("invoking local reboot command");
-
         Command::new("powershell.exe")
             .arg("-Command")
             .arg("Restart-Computer")

diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py
@@ -1281,7 +1281,7 @@ def get_config(self, pool_name: primitives.PoolName) -> models.AgentConfig:
                 client_secret="<client_secret>",
                 resource=self.onefuzz._backend.config.endpoint,
                 tenant=urlparse(self.onefuzz._backend.config.authority).path.strip("/"),
-                multi_tenant_domain=self.onefuzz._backend.config.tenant_domain,
+                multi_tenant_domain=self.onefuzz._backend.config.get_multi_tenant_domain(),
             )
 
         return pool.config

diff --git a/src/cli/onefuzz/backend.py b/src/cli/onefuzz/backend.py
@@ -98,6 +98,12 @@ class BackendConfig(BaseModel):
     features: Set[str] = Field(default_factory=set)
     tenant_domain: str
 
+    def get_multi_tenant_domain(self) -> Optional[str]:
+        if "https://login.microsoftonline.com/common" in self.authority:
+            return self.tenant_domain
+        else:
+            return None
+
 
 class Backend:
     def __init__(
@@ -182,10 +188,11 @@ def get_access_token(self) -> Any:
         if not self.config.endpoint:
             raise Exception("endpoint not configured")
 
-        if "https://login.microsoftonline.com/common" in self.config.authority:
+        multi_tenant_domain = self.config.get_multi_tenant_domain()
+        if multi_tenant_domain is not None:
             endpoint = urlparse(self.config.endpoint).netloc.split(".")[0]
             scopes = [
-                f"api://{self.config.tenant_domain}/{endpoint}/.default",
+                f"api://{multi_tenant_domain}/{endpoint}/.default",
             ]
         else:
             netloc = urlparse(self.config.endpoint).netloc

diff --git a/src/deployment/deploylib/registration.py b/src/deployment/deploylib/registration.py
@@ -307,12 +307,13 @@ def try_sp_create() -> None:
         error: Optional[Exception] = None
         for _ in range(10):
             try:
-                query_microsoft_graph(
+                service_principal = query_microsoft_graph(
                     method="POST",
                     resource="servicePrincipals",
                     body=service_principal_params,
                     subscription=subscription_id,
                 )
+                logger.info(f"created service principal:\n {service_principal}")
                 return
             except GraphQueryError as err:
                 # work around timing issue when creating service principal
@@ -654,8 +655,11 @@ def assign_instance_app_role(
 
     if len(onefuzz_service_principals) == 0:
         raise Exception("onefuzz app service principal not found")
-    onefuzz_service_principal = onefuzz_service_principals[0]
 
+    onefuzz_service_principal = onefuzz_service_principals[0]
+    logger.info(
+        f"Assigning app role instance service principal {onefuzz_service_principal['id']}"
+    )
     if isinstance(application_name, str):
         application_service_principals = query_microsoft_graph_list(
             method="GET",