diff --git a/contrib/deploy-onefuzz-via-azure-devops/Pipfile.lock b/contrib/deploy-onefuzz-via-azure-devops/Pipfile.lock index 2582e58b54..0add23c678 100644 --- a/contrib/deploy-onefuzz-via-azure-devops/Pipfile.lock +++ b/contrib/deploy-onefuzz-via-azure-devops/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "95312c1bc42ac740f5eb0a21649b54c049c4f0f3fa8d1e20e819e74a8cca60fa" + "sha256": "a55afa925ca3583161e22204ddaad54516acbf8fa6513e38259341e633a7f8e4" }, "pipfile-spec": 6, "requires": { @@ -18,11 +18,11 @@ "default": { "astroid": { "hashes": [ - "sha256:14ffbb4f6aa2cf474a0834014005487f7ecd8924996083ab411e7fa0b508ce0b", - "sha256:f4e4ec5294c4b07ac38bab9ca5ddd3914d4bf46f9006eb5c0ae755755061044e" + "sha256:86b0a340a512c65abf4368b80252754cda17c02cdbbd3f587dddf98112233e7b", + "sha256:bb24615c77f4837c707669d16907331374ae8a964650a66999da3f5ca68dc946" ], "markers": "python_full_version >= '3.6.2'", - "version": "==2.11.5" + "version": "==2.11.7" }, "black": { "hashes": [ @@ -55,11 +55,11 @@ }, "certifi": { "hashes": [ - "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7", - "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a" + "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3", + "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18" ], - "markers": "python_version >= '3.6'", - "version": "==2022.5.18.1" + "index": "pypi", + "version": "==2022.12.7" }, "charset-normalizer": { "hashes": [ @@ -79,70 +79,52 @@ }, "dill": { "hashes": [ - "sha256:33501d03270bbe410c72639b350e941882a8b0fd55357580fbc873fba0c59302", - "sha256:d75e41f3eff1eee599d738e76ba8f4ad98ea229db8b085318aa2b3333a208c86" + "sha256:a07ffd2351b8c678dfc4a856a3005f8067aea51d6ba6c700796a4d9e280f39f0", + "sha256:e5db55f3687856d8fbdab002ed78544e1c4559a130302693d839dfe8f93f2373" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'", - "version": "==0.3.5.1" + "markers": "python_version >= '3.7'", + "version": "==0.3.6" }, "idna": { "hashes": [ - "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", - "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" + "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", + "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" ], "markers": "python_version >= '3'", - "version": "==3.3" + "version": "==3.4" }, "isort": { "hashes": [ - "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7", - "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951" + "sha256:7c5bd998504826b6f1e6f2f98b533976b066baba29b8bae83fdeefd0b89c6b70", + "sha256:bf02c95f1fe615ebbe13a619cfed1619ddfe8941274c9e3de3143adca406cb02" ], - "markers": "python_version < '4' and python_full_version >= '3.6.1'", - "version": "==5.10.1" + "markers": "python_version >= '3.7'", + "version": "==5.11.1" }, "lazy-object-proxy": { "hashes": [ - "sha256:043651b6cb706eee4f91854da4a089816a6606c1428fd391573ef8cb642ae4f7", - "sha256:07fa44286cda977bd4803b656ffc1c9b7e3bc7dff7d34263446aec8f8c96f88a", - "sha256:12f3bb77efe1367b2515f8cb4790a11cffae889148ad33adad07b9b55e0ab22c", - "sha256:2052837718516a94940867e16b1bb10edb069ab475c3ad84fd1e1a6dd2c0fcfc", - "sha256:2130db8ed69a48a3440103d4a520b89d8a9405f1b06e2cc81640509e8bf6548f", - "sha256:39b0e26725c5023757fc1ab2a89ef9d7ab23b84f9251e28f9cc114d5b59c1b09", - "sha256:46ff647e76f106bb444b4533bb4153c7370cdf52efc62ccfc1a28bdb3cc95442", - "sha256:4dca6244e4121c74cc20542c2ca39e5c4a5027c81d112bfb893cf0790f96f57e", - "sha256:553b0f0d8dbf21890dd66edd771f9b1b5f51bd912fa5f26de4449bfc5af5e029", - "sha256:677ea950bef409b47e51e733283544ac3d660b709cfce7b187f5ace137960d61", - "sha256:6a24357267aa976abab660b1d47a34aaf07259a0c3859a34e536f1ee6e76b5bb", - "sha256:6a6e94c7b02641d1311228a102607ecd576f70734dc3d5e22610111aeacba8a0", - "sha256:6aff3fe5de0831867092e017cf67e2750c6a1c7d88d84d2481bd84a2e019ec35", - "sha256:6ecbb350991d6434e1388bee761ece3260e5228952b1f0c46ffc800eb313ff42", - "sha256:7096a5e0c1115ec82641afbdd70451a144558ea5cf564a896294e346eb611be1", - "sha256:70ed0c2b380eb6248abdef3cd425fc52f0abd92d2b07ce26359fcbc399f636ad", - "sha256:8561da8b3dd22d696244d6d0d5330618c993a215070f473b699e00cf1f3f6443", - "sha256:85b232e791f2229a4f55840ed54706110c80c0a210d076eee093f2b2e33e1bfd", - "sha256:898322f8d078f2654d275124a8dd19b079080ae977033b713f677afcfc88e2b9", - "sha256:8f3953eb575b45480db6568306893f0bd9d8dfeeebd46812aa09ca9579595148", - "sha256:91ba172fc5b03978764d1df5144b4ba4ab13290d7bab7a50f12d8117f8630c38", - "sha256:9d166602b525bf54ac994cf833c385bfcc341b364e3ee71e3bf5a1336e677b55", - "sha256:a57d51ed2997e97f3b8e3500c984db50a554bb5db56c50b5dab1b41339b37e36", - "sha256:b9e89b87c707dd769c4ea91f7a31538888aad05c116a59820f28d59b3ebfe25a", - "sha256:bb8c5fd1684d60a9902c60ebe276da1f2281a318ca16c1d0a96db28f62e9166b", - "sha256:c19814163728941bb871240d45c4c30d33b8a2e85972c44d4e63dd7107faba44", - "sha256:c4ce15276a1a14549d7e81c243b887293904ad2d94ad767f42df91e75fd7b5b6", - "sha256:c7a683c37a8a24f6428c28c561c80d5f4fd316ddcf0c7cab999b15ab3f5c5c69", - "sha256:d609c75b986def706743cdebe5e47553f4a5a1da9c5ff66d76013ef396b5a8a4", - "sha256:d66906d5785da8e0be7360912e99c9188b70f52c422f9fc18223347235691a84", - "sha256:dd7ed7429dbb6c494aa9bc4e09d94b778a3579be699f9d67da7e6804c422d3de", - "sha256:df2631f9d67259dc9620d831384ed7732a198eb434eadf69aea95ad18c587a28", - "sha256:e368b7f7eac182a59ff1f81d5f3802161932a41dc1b1cc45c1f757dc876b5d2c", - "sha256:e40f2013d96d30217a51eeb1db28c9ac41e9d0ee915ef9d00da639c5b63f01a1", - "sha256:f769457a639403073968d118bc70110e7dce294688009f5c24ab78800ae56dc8", - "sha256:fccdf7c2c5821a8cbd0a9440a456f5050492f2270bd54e94360cac663398739b", - "sha256:fd45683c3caddf83abbb1249b653a266e7069a09f486daa8863fb0e7496a9fdb" + "sha256:0c1c7c0433154bb7c54185714c6929acc0ba04ee1b167314a779b9025517eada", + "sha256:14010b49a2f56ec4943b6cf925f597b534ee2fe1f0738c84b3bce0c1a11ff10d", + "sha256:4e2d9f764f1befd8bdc97673261b8bb888764dfdbd7a4d8f55e4fbcabb8c3fb7", + "sha256:4fd031589121ad46e293629b39604031d354043bb5cdf83da4e93c2d7f3389fe", + "sha256:5b51d6f3bfeb289dfd4e95de2ecd464cd51982fe6f00e2be1d0bf94864d58acd", + "sha256:6850e4aeca6d0df35bb06e05c8b934ff7c533734eb51d0ceb2d63696f1e6030c", + "sha256:6f593f26c470a379cf7f5bc6db6b5f1722353e7bf937b8d0d0b3fba911998858", + "sha256:71d9ae8a82203511a6f60ca5a1b9f8ad201cac0fc75038b2dc5fa519589c9288", + "sha256:7e1561626c49cb394268edd00501b289053a652ed762c58e1081224c8d881cec", + "sha256:8f6ce2118a90efa7f62dd38c7dbfffd42f468b180287b748626293bf12ed468f", + "sha256:ae032743794fba4d171b5b67310d69176287b5bf82a21f588282406a79498891", + "sha256:afcaa24e48bb23b3be31e329deb3f1858f1f1df86aea3d70cb5c8578bfe5261c", + "sha256:b70d6e7a332eb0217e7872a73926ad4fdc14f846e85ad6749ad111084e76df25", + "sha256:c219a00245af0f6fa4e95901ed28044544f50152840c5b6a3e7b2568db34d156", + "sha256:ce58b2b3734c73e68f0e30e4e725264d4d6be95818ec0a0be4bb6bf9a7e79aa8", + "sha256:d176f392dbbdaacccf15919c77f526edf11a34aece58b55ab58539807b85436f", + "sha256:e20bfa6db17a39c706d24f82df8352488d2943a3b7ce7d4c22579cb89ca8896e", + "sha256:eac3a9a5ef13b332c059772fd40b4b1c3d45a3a2b05e33a361dee48e54a4dad0", + "sha256:eb329f8d8145379bf5dbe722182410fe8863d186e51bf034d2075eb8d85ee25b" ], - "markers": "python_version >= '3.6'", - "version": "==1.7.1" + "markers": "python_version >= '3.7'", + "version": "==1.8.0" }, "mccabe": { "hashes": [ @@ -161,10 +143,11 @@ }, "pathspec": { "hashes": [ - "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a", - "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1" + "sha256:3c95343af8b756205e2aba76e843ba9520a24dd84f68c22b9f93251507509dd6", + "sha256:56200de4077d9d0791465aa9095a01d421861e405b5096955051deefd697d6f6" ], - "version": "==0.9.0" + "markers": "python_version >= '3.7'", + "version": "==0.10.3" }, "pip": { "hashes": [ @@ -176,11 +159,11 @@ }, "platformdirs": { "hashes": [ - "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788", - "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19" + "sha256:1a89a12377800c81983db6be069ec068eee989748799b946cce2a6e80dcc54ca", + "sha256:b46ffafa316e6b83b47489d240ce17173f123a9b9c83282141c3daf26ad9ac2e" ], "markers": "python_version >= '3.7'", - "version": "==2.5.2" + "version": "==2.6.0" }, "pylint": { "hashes": [ @@ -200,11 +183,11 @@ }, "setuptools": { "hashes": [ - "sha256:68e45d17c9281ba25dc0104eadd2647172b3472d9e01f911efa57965e8d51a36", - "sha256:a43bdedf853c670e5fed28e5623403bad2f73cf02f9a2774e91def6bda8265a7" + "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54", + "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75" ], "markers": "python_version >= '3.7'", - "version": "==62.3.2" + "version": "==65.6.3" }, "tomli": { "hashes": [ @@ -216,19 +199,19 @@ }, "typing-extensions": { "hashes": [ - "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708", - "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376" + "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa", + "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e" ], "markers": "python_version < '3.10'", - "version": "==4.2.0" + "version": "==4.4.0" }, "urllib3": { "hashes": [ - "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14", - "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e" + "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc", + "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.26.9" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==1.26.13" }, "wrapt": { "hashes": [ diff --git a/docs/unmnaged-nodes.md b/docs/unmnaged-nodes.md new file mode 100644 index 0000000000..0850fb0c68 --- /dev/null +++ b/docs/unmnaged-nodes.md @@ -0,0 +1,80 @@ +# Unmanaged Nodes +The default mode of OneFuzz is to run the agents inside scalesets managed by the the Onefuzz instance. But it is possible to run outside of the Instance infrastructure. +This is the unmanaged scenario. In this mode, the user can use their own resource to participate in the fuzzing. + +## Set-up +These are the steps to run an unmanaged node + + +### Create an Application Registration in Azure Active Directory +We will create the authentication method for the unmanaged node. +From the [azure cli](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) create a new **application registration**: +```cmd +az ad app create --display-name +``` +Then use the application `app_id` in the result to create the associated **service principal**: + +```cmd +az ad sp create --id +``` +Take note of the `id` returned by this request. We will call it the `principal_id`. + +Next, create a `client_secret`: + +``` +az ad app credential reset --id --append +``` +Take note of the `password` returned. + +### Authorize the application in OneFuzz +From the OneFuzz `deployment` folder run the following script using the `app_id` from above: +``` cmd +python .\deploylib\registration.py register_app --app_id --role UnmanagedNode +``` + +### Create an unmanaged pool +Using the OneFuzz CLI: +``` cmd +onefuzz pools create --unmanaged --object_id +``` + +### Download the agent binaries and the agent configuration +Download a zip file containing the agent binaries: +``` +onefuzz tools get +``` +Extract the zip file in a folder of your choice. + +Download the configuration file for the agent: + +``` +onefuzz pools get_config +``` + +Under the `client_credential` section of the agent config file, update `client_id` and `client_secret`: +```json +{ + "client_id": "", + "client_secret": "", +} +``` +Save the config to the file. + +### Start the agent. +Navigate to the folder corresponding to your OS. +Set the necessary environment variable by running the script `set-env.ps1` (for Windows) or `set-env.sh` (for Linux). +Run the agent with the following command. If you need more nodes use a different `machine_guid` for each one: +```cmd +onefuzz-agent run --machine_id -c --reset_lock +``` + +### Verify that the agent is registered to OneFuzz + +Using the OneFuzz CLI run the following command: + +``` +onefuzz nodes get +``` + +This should return one entry. Verify that the `pool_name` matched the pool name created earlier. +From here you will be able to schedule jobs on that pool and they will be running. \ No newline at end of file diff --git a/src/ApiService/ApiService/Functions/AgentRegistration.cs b/src/ApiService/ApiService/Functions/AgentRegistration.cs index 0236372d85..ce380c7c78 100644 --- a/src/ApiService/ApiService/Functions/AgentRegistration.cs +++ b/src/ApiService/ApiService/Functions/AgentRegistration.cs @@ -152,7 +152,9 @@ private async Async.Task Post(HttpRequestData req) { MachineId: machineId, ScalesetId: scalesetId, InstanceId: instanceId, - Version: version + Version: version, + Os: os ?? pool.Os, + Managed: pool.Managed ); var r = await _context.NodeOperations.Replace(node); diff --git a/src/ApiService/ApiService/GroupMembershipChecker.cs b/src/ApiService/ApiService/GroupMembershipChecker.cs index 7ffc62c96e..85c21b40ba 100644 --- a/src/ApiService/ApiService/GroupMembershipChecker.cs +++ b/src/ApiService/ApiService/GroupMembershipChecker.cs @@ -15,7 +15,7 @@ public async ValueTask IsMember(IEnumerable groupIds, Guid memberId) } } -class AzureADGroupMembership : GroupMembershipChecker { +sealed class AzureADGroupMembership : GroupMembershipChecker { private readonly GraphServiceClient _graphClient; public AzureADGroupMembership(GraphServiceClient graphClient) => _graphClient = graphClient; protected override async IAsyncEnumerable GetGroups(Guid memberId) { @@ -30,7 +30,7 @@ protected override async IAsyncEnumerable GetGroups(Guid memberId) { } } -class StaticGroupMembership : GroupMembershipChecker { +sealed class StaticGroupMembership : GroupMembershipChecker { private readonly IReadOnlyDictionary> _memberships; public StaticGroupMembership(IDictionary memberships) { _memberships = memberships.ToDictionary(kvp => kvp.Key, kvp => (IReadOnlyList)kvp.Value.ToList()); diff --git a/src/ApiService/ApiService/Log.cs b/src/ApiService/ApiService/Log.cs index 5c00175c6f..76435890e0 100644 --- a/src/ApiService/ApiService/Log.cs +++ b/src/ApiService/ApiService/Log.cs @@ -61,7 +61,7 @@ public interface ILog { void Flush(); } -class AppInsights : ILog { +sealed class AppInsights : ILog { private readonly TelemetryClient _telemetryClient; public AppInsights(TelemetryClient client) { @@ -128,7 +128,7 @@ public void Flush() { } //TODO: Should we write errors and Exception to std err ? -class Console : ILog { +sealed class Console : ILog { private static string DictToString(IReadOnlyDictionary? d) { if (d is null) { diff --git a/src/ApiService/ApiService/OneFuzzTypes/Events.cs b/src/ApiService/ApiService/OneFuzzTypes/Events.cs index 4257a7baaf..7273bb9e7f 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Events.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Events.cs @@ -122,7 +122,7 @@ TaskConfig Config [EventType(EventType.JobCreated)] -record EventJobCreated( +public record EventJobCreated( Guid JobId, JobConfig Config, UserInfo? UserInfo @@ -146,7 +146,7 @@ List TaskInfo [EventType(EventType.TaskCreated)] -record EventTaskCreated( +public record EventTaskCreated( Guid JobId, Guid TaskId, TaskConfig Config, @@ -177,7 +177,7 @@ Guid PingId [EventType(EventType.ScalesetCreated)] -record EventScalesetCreated( +public record EventScalesetCreated( Guid ScalesetId, PoolName PoolName, string VmSku, @@ -187,7 +187,7 @@ record EventScalesetCreated( [EventType(EventType.ScalesetFailed)] -public record EventScalesetFailed( +public sealed record EventScalesetFailed( Guid ScalesetId, PoolName PoolName, Error Error @@ -195,7 +195,7 @@ Error Error [EventType(EventType.ScalesetDeleted)] -record EventScalesetDeleted( +public record EventScalesetDeleted( Guid ScalesetId, PoolName PoolName @@ -211,13 +211,13 @@ long size [EventType(EventType.PoolDeleted)] -record EventPoolDeleted( +public record EventPoolDeleted( PoolName PoolName ) : BaseEvent(); [EventType(EventType.PoolCreated)] -record EventPoolCreated( +public record EventPoolCreated( PoolName PoolName, Os Os, Architecture Arch, @@ -289,7 +289,7 @@ ScalesetState State ) : BaseEvent(); [EventType(EventType.NodeStateUpdated)] -record EventNodeStateUpdated( +public record EventNodeStateUpdated( Guid MachineId, Guid? ScalesetId, PoolName PoolName, diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index ff274f0a31..83ed622734 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -110,7 +110,8 @@ public record Node bool ReimageRequested = false, bool DeleteRequested = false, - bool DebugKeepNode = false + bool DebugKeepNode = false, + bool Managed = true ) : StatefulEntityBase(State) { public List? Tasks { get; set; } diff --git a/src/ApiService/ApiService/ServiceConfiguration.cs b/src/ApiService/ApiService/ServiceConfiguration.cs index 57f056122c..9b50c53f23 100644 --- a/src/ApiService/ApiService/ServiceConfiguration.cs +++ b/src/ApiService/ApiService/ServiceConfiguration.cs @@ -32,6 +32,7 @@ public interface IServiceConfig { public ResourceIdentifier? OneFuzzFuncStorage { get; } public string? OneFuzzInstance { get; } public string? OneFuzzInstanceName { get; } + public string? OneFuzzEndpoint { get; } public string? OneFuzzKeyvault { get; } public string? OneFuzzMonitor { get; } @@ -117,6 +118,7 @@ public ResourceIdentifier? OneFuzzFuncStorage { public string? OneFuzzInstance { get => GetEnv("ONEFUZZ_INSTANCE"); } public string? OneFuzzInstanceName { get => GetEnv("ONEFUZZ_INSTANCE_NAME"); } + public string? OneFuzzEndpoint { get => GetEnv("ONEFUZZ_ENDPOINT"); } public string? OneFuzzKeyvault { get => GetEnv("ONEFUZZ_KEYVAULT"); } public string? OneFuzzMonitor { get => GetEnv("ONEFUZZ_MONITOR"); } public string? OneFuzzOwner { get => GetEnv("ONEFUZZ_OWNER"); } diff --git a/src/ApiService/ApiService/TestHooks/NodeOperationsTestHooks.cs b/src/ApiService/ApiService/TestHooks/NodeOperationsTestHooks.cs index 90f0875d69..29234e2d2a 100644 --- a/src/ApiService/ApiService/TestHooks/NodeOperationsTestHooks.cs +++ b/src/ApiService/ApiService/TestHooks/NodeOperationsTestHooks.cs @@ -10,7 +10,7 @@ #if DEBUG namespace ApiService.TestHooks { - record MarkTasks(Node node, Error? error); + sealed record MarkTasks(Node node, Error? error); public class NodeOperationsTestHooks { private readonly ILogTracer _log; @@ -162,8 +162,8 @@ public async Task SearchStates([HttpTrigger(AuthorizationLevel Guid? scaleSetId = UriExtension.GetGuid("scaleSetId", query); List? states = default; - if (query.ContainsKey("states")) { - states = query["states"].Split('-').Select(s => Enum.Parse(s)).ToList(); + if (query.TryGetValue("states", out var value)) { + states = value.Split('-').Select(s => Enum.Parse(s)).ToList(); } string? poolNameString = UriExtension.GetString("poolName", query); @@ -214,8 +214,8 @@ public async Task CreateNode([HttpTrigger(AuthorizationLevel.A Guid machineId = Guid.Parse(query["machineId"]); Guid? scaleSetId = default; - if (query.ContainsKey("scaleSetId")) { - scaleSetId = Guid.Parse(query["scaleSetId"]); + if (query.TryGetValue("scaleSetId", out var value)) { + scaleSetId = Guid.Parse(value); } string version = query["version"]; diff --git a/src/ApiService/ApiService/TestHooks/UriExtension.cs b/src/ApiService/ApiService/TestHooks/UriExtension.cs index 4b14a922b3..07b4329521 100644 --- a/src/ApiService/ApiService/TestHooks/UriExtension.cs +++ b/src/ApiService/ApiService/TestHooks/UriExtension.cs @@ -15,8 +15,8 @@ from cs in queryComponents public static bool GetBool(string key, IDictionary query, bool defaultValue = false) { bool v; - if (query.ContainsKey(key)) { - v = bool.Parse(query[key]); + if (query.TryGetValue(key, out var value)) { + v = bool.Parse(value); } else { v = defaultValue; } @@ -25,8 +25,8 @@ public static bool GetBool(string key, IDictionary query, bool d public static int? GetInt(string key, IDictionary query, int? defaultValue = null) { int? v; - if (query.ContainsKey(key)) { - v = int.Parse(query[key]); + if (query.TryGetValue(key, out var value)) { + v = int.Parse(value); } else { v = defaultValue; } @@ -35,16 +35,16 @@ public static bool GetBool(string key, IDictionary query, bool d public static string? GetString(string key, IDictionary query, string? defaultValue = null) { - if (query.ContainsKey(key)) { - return query[key]; + if (query.TryGetValue(key, out var value)) { + return value; } else { return defaultValue; } } public static Guid? GetGuid(string key, IDictionary query, Guid? defaultValue = null) { - if (query.ContainsKey(key)) { - return Guid.Parse(query[key]); + if (query.TryGetValue(key, out var value)) { + return Guid.Parse(value); } else { return defaultValue; } diff --git a/src/ApiService/ApiService/onefuzzlib/Config.cs b/src/ApiService/ApiService/onefuzzlib/Config.cs index 5c045b34b3..be5fea0fc1 100644 --- a/src/ApiService/ApiService/onefuzzlib/Config.cs +++ b/src/ApiService/ApiService/onefuzzlib/Config.cs @@ -505,8 +505,7 @@ private async Task> CheckContainers(TaskDefinition d } private static ResultVoid CheckContainer(Compare compare, long expected, ContainerType containerType, Dictionary> containers) { - var actual = containers.ContainsKey(containerType) ? containers[containerType].Count : 0; - + var actual = containers.TryGetValue(containerType, out var v) ? v.Count : 0; if (!CheckVal(compare, expected, actual)) { return ResultVoid.Error( new TaskConfigError($"container type {containerType}: expected {compare} {expected}, got {actual}")); diff --git a/src/ApiService/ApiService/onefuzzlib/Creds.cs b/src/ApiService/ApiService/onefuzzlib/Creds.cs index 85b350e76c..5e085861f5 100644 --- a/src/ApiService/ApiService/onefuzzlib/Creds.cs +++ b/src/ApiService/ApiService/onefuzzlib/Creds.cs @@ -105,8 +105,10 @@ public Async.Task GetBaseRegion() { }); } - public Uri GetInstanceUrl() - => new($"https://{GetInstanceName()}.azurewebsites.net"); + public Uri GetInstanceUrl() { + var onefuzzEndpoint = _config.OneFuzzEndpoint; + return onefuzzEndpoint != null ? new Uri(onefuzzEndpoint) : new($"https://{GetInstanceName()}.azurewebsites.net"); + } public record ScaleSetIdentity(string principalId); @@ -159,7 +161,7 @@ public Task> GetRegions() } -class GraphQueryException : Exception { +sealed class GraphQueryException : Exception { public GraphQueryException(string? message) : base(message) { } } diff --git a/src/ApiService/ApiService/onefuzzlib/Extension.cs b/src/ApiService/ApiService/onefuzzlib/Extension.cs index bd5bbe75bf..29a3bfbd7d 100644 --- a/src/ApiService/ApiService/onefuzzlib/Extension.cs +++ b/src/ApiService/ApiService/onefuzzlib/Extension.cs @@ -130,7 +130,7 @@ public static VMExtensionWrapper AzSecExtension(AzureLocation region) { }; } - private class Settings { + private sealed class Settings { [JsonPropertyName("GCS_AUTO_CONFIG")] public bool GCS_AUTO_CONFIG { get; set; } = true; } diff --git a/src/ApiService/ApiService/onefuzzlib/ImageOperations.cs b/src/ApiService/ApiService/onefuzzlib/ImageOperations.cs index 89c39d4449..21c6618e55 100644 --- a/src/ApiService/ApiService/onefuzzlib/ImageOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/ImageOperations.cs @@ -39,7 +39,7 @@ public ImageOperations(ILogTracer logTracer, IOnefuzzContext context, IMemoryCac _cache = cache; } - record class GetOsKey(Region Region, string Image); + sealed record class GetOsKey(Region Region, string Image); public Task> GetOs(Region region, string image) => _cache.GetOrCreateAsync>(new GetOsKey(region, image), entry => GetOsInternal(region, image)); diff --git a/src/ApiService/ApiService/onefuzzlib/IpOperations.cs b/src/ApiService/ApiService/onefuzzlib/IpOperations.cs index 3356de551a..4158a4da15 100644 --- a/src/ApiService/ApiService/onefuzzlib/IpOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/IpOperations.cs @@ -232,16 +232,16 @@ public async Async.Task CreateIp(string resourceGroup, string name, Region regio /// the api does not seems to support this : /// https://github.com/Azure/azure-sdk-for-net/issues/30253#issuecomment-1202447362 /// - class NetworkInterfaceQuery { - record IpConfigurationsProperties(string privateIPAddress); + sealed class NetworkInterfaceQuery { + sealed record IpConfigurationsProperties(string privateIPAddress); - record IpConfigurations(IpConfigurationsProperties properties); + sealed record IpConfigurations(IpConfigurationsProperties properties); - record NetworkInterfaceProperties(List ipConfigurations); + sealed record NetworkInterfaceProperties(List ipConfigurations); - record NetworkInterface(NetworkInterfaceProperties properties); + sealed record NetworkInterface(NetworkInterfaceProperties properties); - record ValueList(List value); + sealed record ValueList(List value); private readonly IOnefuzzContext _context; diff --git a/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs b/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs index a56a5ad073..b3e7acae83 100644 --- a/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs @@ -144,7 +144,7 @@ await TryGetNodeInfo(node) is NodeInfo nodeInfo) { return OneFuzzResultVoid.Ok; } - record NodeInfo(Node Node, Scaleset Scaleset, string InstanceId); + sealed record NodeInfo(Node Node, Scaleset Scaleset, string InstanceId); private async Async.Task TryGetNodeInfo(Node node) { var scalesetId = node.ScalesetId; if (scalesetId is null) { @@ -337,6 +337,10 @@ public async Async.Task CleanupBusyNodesWithoutWork() { } public async Async.Task ToReimage(Node node, bool done = false) { + if (!node.Managed) { + _logTracer.Info($"skip reimage for unmanaged node: {node.MachineId:Tag:MachineId}"); + return node; + } var nodeState = node.State; if (done) { diff --git a/src/ApiService/ApiService/onefuzzlib/RequestAccess.cs b/src/ApiService/ApiService/onefuzzlib/RequestAccess.cs index c242f4e8e9..b228ba0285 100644 --- a/src/ApiService/ApiService/onefuzzlib/RequestAccess.cs +++ b/src/ApiService/ApiService/onefuzzlib/RequestAccess.cs @@ -7,7 +7,7 @@ public class RequestAccess { private readonly Node _root = new(); public record Rules(IReadOnlyList AllowedGroupsIds); - record Node( + sealed record Node( // HTTP Method -> Rules Dictionary Rules, // Path Segment -> Node diff --git a/src/ApiService/ApiService/onefuzzlib/Scheduler.cs b/src/ApiService/ApiService/onefuzzlib/Scheduler.cs index ebbc2083c9..cfce46bc1f 100644 --- a/src/ApiService/ApiService/onefuzzlib/Scheduler.cs +++ b/src/ApiService/ApiService/onefuzzlib/Scheduler.cs @@ -118,9 +118,9 @@ private async Async.Task ScheduleWorkset(WorkSet workSet, Pool pool, long } - record BucketConfig(long count, bool reboot, Container setupContainer, string? setupScript, Pool pool); + sealed record BucketConfig(long count, bool reboot, Container setupContainer, string? setupScript, Pool pool); - record PoolKey( + sealed record PoolKey( PoolName? poolName = null, (string sku, string image)? vm = null); diff --git a/src/ApiService/ApiService/onefuzzlib/Storage.cs b/src/ApiService/ApiService/onefuzzlib/Storage.cs index e272baf78f..33541a43e4 100644 --- a/src/ApiService/ApiService/onefuzzlib/Storage.cs +++ b/src/ApiService/ApiService/onefuzzlib/Storage.cs @@ -157,7 +157,7 @@ public ResourceIdentifier GetPrimaryAccount(StorageType storageType) var x => throw new NotSupportedException($"invalid StorageType: {x}"), }; - record GetStorageAccountKey_CacheKey(ResourceIdentifier Identifier); + sealed record GetStorageAccountKey_CacheKey(ResourceIdentifier Identifier); public Async.Task GetStorageAccountKey(string accountName) { var resourceGroupId = _creds.GetResourceGroupResourceIdentifier(); var storageAccountId = StorageAccountResource.CreateResourceIdentifier(resourceGroupId.SubscriptionId, resourceGroupId.Name, accountName); @@ -187,7 +187,7 @@ private static Uri GetBlobEndpoint(string accountName) // According to guidance these should be reused as they manage HttpClients, // so we cache them all by account: - record BlobClientKey(string AccountName); + sealed record BlobClientKey(string AccountName); public Task GetBlobServiceClientForAccountName(string accountName) { return _cache.GetOrCreate(new BlobClientKey(accountName), async cacheEntry => { cacheEntry.Priority = CacheItemPriority.NeverRemove; @@ -197,7 +197,7 @@ public Task GetBlobServiceClientForAccountName(string account }); } - record TableClientKey(string AccountName); + sealed record TableClientKey(string AccountName); public Task GetTableServiceClientForAccountName(string accountName) => _cache.GetOrCreate(new TableClientKey(accountName), async cacheEntry => { cacheEntry.Priority = CacheItemPriority.NeverRemove; @@ -206,7 +206,7 @@ public Task GetTableServiceClientForAccountName(string accou return new TableServiceClient(GetTableEndpoint(accountName), skc); }); - record QueueClientKey(string AccountName); + sealed record QueueClientKey(string AccountName); private static readonly QueueClientOptions _queueClientOptions = new() { MessageEncoding = QueueMessageEncoding.Base64 }; public Task GetQueueServiceClientForAccountName(string accountName) => _cache.GetOrCreateAsync(new QueueClientKey(accountName), async cacheEntry => { diff --git a/src/ApiService/ApiService/onefuzzlib/VmssOperations.cs b/src/ApiService/ApiService/onefuzzlib/VmssOperations.cs index 84d210fd8e..a283382827 100644 --- a/src/ApiService/ApiService/onefuzzlib/VmssOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/VmssOperations.cs @@ -187,7 +187,7 @@ public async Async.Task> ListInstanceIds(Guid name) { } } - private record InstanceIdKey(Guid Scaleset, Guid VmId); + private sealed record InstanceIdKey(Guid Scaleset, Guid VmId); private Task GetInstanceIdForVmId(Guid scaleset, Guid vmId) => _cache.GetOrCreateAsync(new InstanceIdKey(scaleset, vmId), async entry => { var scalesetResource = GetVmssResource(scaleset); diff --git a/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs b/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs index 64ddfafa98..5ded817f51 100644 --- a/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs +++ b/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs @@ -56,7 +56,7 @@ private static bool IsTransient(Exception e) { return errorCodes.Any(code => errorStr.Contains(code)); } - class AdoConnector { + sealed class AdoConnector { private readonly AdoTemplate _config; private readonly Renderer _renderer; private readonly string _project; @@ -187,7 +187,7 @@ public async Async.Task UpdateExisting(WorkItem item, (string, string)[] n var document = new JsonPatchDocument(); foreach (var field in _config.OnDuplicate.Increment) { - var value = item.Fields.ContainsKey(field) ? int.Parse(JsonSerializer.Serialize(item.Fields[field])) : 0; + var value = item.Fields.TryGetValue(field, out var fieldValue) ? int.Parse(JsonSerializer.Serialize(fieldValue)) : 0; value++; document.Add(new JsonPatchOperation() { Operation = VisualStudio.Services.WebApi.Patch.Operation.Replace, @@ -207,11 +207,11 @@ public async Async.Task UpdateExisting(WorkItem item, (string, string)[] n var systemState = JsonSerializer.Serialize(item.Fields["System.State"]); var stateUpdated = false; - if (_config.OnDuplicate.SetState.ContainsKey(systemState)) { + if (_config.OnDuplicate.SetState.TryGetValue(systemState, out var v)) { document.Add(new JsonPatchOperation() { Operation = VisualStudio.Services.WebApi.Patch.Operation.Replace, Path = "/fields/System.State", - Value = _config.OnDuplicate.SetState[systemState] + Value = v }); stateUpdated = true; diff --git a/src/ApiService/ApiService/onefuzzlib/notifications/GithubIssues.cs b/src/ApiService/ApiService/onefuzzlib/notifications/GithubIssues.cs index 4c5698b88e..90dbc3cc97 100644 --- a/src/ApiService/ApiService/onefuzzlib/notifications/GithubIssues.cs +++ b/src/ApiService/ApiService/onefuzzlib/notifications/GithubIssues.cs @@ -35,7 +35,7 @@ private async Async.Task Process(GithubIssuesTemplate config, Container containe var handler = await GithubConnnector.GithubConnnectorCreator(config, container, filename, renderer, _context.Creds.GetInstanceUrl(), _context, _logTracer); await handler.Process(); } - class GithubConnnector { + sealed class GithubConnnector { private readonly GitHubClient _gh; private readonly GithubIssuesTemplate _config; private readonly Renderer _renderer; diff --git a/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs b/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs index 4b04caea26..4ed195579d 100644 --- a/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs +++ b/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs @@ -81,7 +81,7 @@ ParameterInfo parameterInfo ); public record EntityInfo(Type type, ILookup properties, Func constructor); -class OnefuzzNamingPolicy : JsonNamingPolicy { +sealed class OnefuzzNamingPolicy : JsonNamingPolicy { public override string ConvertName(string name) { return CaseConverter.PascalToSnake(name); } diff --git a/src/ApiService/FunctionalTests/1f-api/ApiClient.cs b/src/ApiService/FunctionalTests/1f-api/ApiClient.cs index 290e24d02c..2055ee9215 100644 --- a/src/ApiService/FunctionalTests/1f-api/ApiClient.cs +++ b/src/ApiService/FunctionalTests/1f-api/ApiClient.cs @@ -1,5 +1,5 @@ namespace FunctionalTests { - class ApiClient { + sealed class ApiClient { static Uri endpoint = new Uri(System.Environment.GetEnvironmentVariable("ONEFUZZ_ENDPOINT") ?? "http://localhost:7071"); static Microsoft.Morse.AuthenticationConfig authConfig = diff --git a/src/ApiService/FunctionalTests/1f-api/Info.cs b/src/ApiService/FunctionalTests/1f-api/Info.cs index e8859a18e8..2ab2deb822 100644 --- a/src/ApiService/FunctionalTests/1f-api/Info.cs +++ b/src/ApiService/FunctionalTests/1f-api/Info.cs @@ -30,7 +30,7 @@ public class InfoResponse : IFromJsonElement { } - class InfoApi : ApiBase { + sealed class InfoApi : ApiBase { public InfoApi(Uri endpoint, Microsoft.OneFuzz.Service.Request request, ITestOutputHelper output) : base(endpoint, "/api/Info", request, output) { diff --git a/src/ApiService/IntegrationTests/Fakes/TestContainers.cs b/src/ApiService/IntegrationTests/Fakes/TestContainers.cs index 7422999948..f405dab22d 100644 --- a/src/ApiService/IntegrationTests/Fakes/TestContainers.cs +++ b/src/ApiService/IntegrationTests/Fakes/TestContainers.cs @@ -3,7 +3,7 @@ using Microsoft.OneFuzz.Service; // TestContainers class allows use of InstanceID without having to set it up in blob storage -class TestContainers : Containers { +sealed class TestContainers : Containers { public TestContainers(ILogTracer log, IStorage storage, IServiceConfig config) : base(log, storage, config) { } diff --git a/src/ApiService/IntegrationTests/Fakes/TestCreds.cs b/src/ApiService/IntegrationTests/Fakes/TestCreds.cs index 889dabece6..542c2bebe7 100644 --- a/src/ApiService/IntegrationTests/Fakes/TestCreds.cs +++ b/src/ApiService/IntegrationTests/Fakes/TestCreds.cs @@ -11,7 +11,7 @@ namespace IntegrationTests.Fakes; -class TestCreds : ICreds { +sealed class TestCreds : ICreds { private readonly Guid _subscriptionId; private readonly string _resourceGroup; diff --git a/src/ApiService/IntegrationTests/Fakes/TestServiceConfiguration.cs b/src/ApiService/IntegrationTests/Fakes/TestServiceConfiguration.cs index 72a1622d44..eb6d487e8e 100644 --- a/src/ApiService/IntegrationTests/Fakes/TestServiceConfiguration.cs +++ b/src/ApiService/IntegrationTests/Fakes/TestServiceConfiguration.cs @@ -31,6 +31,8 @@ public TestServiceConfiguration(string tablePrefix) { // -- Remainder not implemented -- + public string? OneFuzzEndpoint => throw new System.NotImplementedException(); + public LogDestination[] LogDestinations { get => throw new System.NotImplementedException(); set => throw new System.NotImplementedException(); } public SeverityLevel LogSeverityLevel => throw new System.NotImplementedException(); diff --git a/src/ApiService/Tests/OrmTest.cs b/src/ApiService/Tests/OrmTest.cs index 0724b8977b..5c8f55e712 100644 --- a/src/ApiService/Tests/OrmTest.cs +++ b/src/ApiService/Tests/OrmTest.cs @@ -10,7 +10,7 @@ namespace Tests { public class OrmTest { - class TestObject { + sealed class TestObject { public String? TheName { get; set; } public TestEnum TheEnum { get; set; } public TestFlagEnum TheFlag { get; set; } @@ -34,7 +34,7 @@ enum TestEnumValue { Two = 2 } - record Entity1( + sealed record Entity1( [PartitionKey] Guid Id, [RowKey] string TheName, DateTimeOffset TheDate, @@ -241,7 +241,7 @@ public void TestEventSerialization() { } - record Entity2( + sealed record Entity2( [PartitionKey] int Id, [RowKey] string TheName ) : EntityBase(); @@ -257,7 +257,7 @@ public void TestIntKey() { Assert.Equal(expected.TheName, actual.TheName); } - record Entity3( + sealed record Entity3( [PartitionKey] int Id, [RowKey] string TheName, Container Container @@ -292,7 +292,7 @@ public void TestContainerSerialization2() { } - record Entity4( + sealed record Entity4( [RowKey][PartitionKey] int Id, string TheName, Container Container @@ -315,7 +315,7 @@ public void TestPartitionKeyIsRowKey() { } - record TestEnumObject(TestEnumValue TheEnumValue); + sealed record TestEnumObject(TestEnumValue TheEnumValue); [Fact] public void TestSerializeEnumValue() { @@ -331,7 +331,7 @@ public void TestSerializeEnumValue() { } - record TestNullField(int? Id, string? Name, TestObject? Obj) : EntityBase(); + sealed record TestNullField(int? Id, string? Name, TestObject? Obj) : EntityBase(); [Fact] public void TestNullValue() { @@ -361,7 +361,7 @@ enum DoNotRenameFlag { Test_2 = 1 << 1, TEST3 = 1 << 2, } - record TestEntity3(DoNotRename Enum, DoNotRenameFlag flag) : EntityBase(); + sealed record TestEntity3(DoNotRename Enum, DoNotRenameFlag flag) : EntityBase(); [Fact] @@ -380,13 +380,13 @@ public void TestSkipRename() { } - class TestClass { + sealed class TestClass { public string Name { get; } public TestClass() { Name = "testName"; } } - record TestIinit([DefaultValue(InitMethod.DefaultConstructor)] TestClass testClass, string test = "default_test") : EntityBase(); + sealed record TestIinit([DefaultValue(InitMethod.DefaultConstructor)] TestClass testClass, string test = "default_test") : EntityBase(); [Fact] public void TestInitValue() { @@ -399,7 +399,7 @@ public void TestInitValue() { } - record TestKeyGetter([PartitionKey] Guid PartitionKey, [RowKey] Guid RowKey); + sealed record TestKeyGetter([PartitionKey] Guid PartitionKey, [RowKey] Guid RowKey); [Fact] public void TestKeyGetters() { var test = new TestKeyGetter(Guid.NewGuid(), Guid.NewGuid()); diff --git a/src/ApiService/Tests/ValidatedStringTests.cs b/src/ApiService/Tests/ValidatedStringTests.cs index 645c1bbb2b..c839b12266 100644 --- a/src/ApiService/Tests/ValidatedStringTests.cs +++ b/src/ApiService/Tests/ValidatedStringTests.cs @@ -6,7 +6,7 @@ namespace Tests; public class ValidatedStringTests { - record ThingContainingPoolName(PoolName PoolName); + sealed record ThingContainingPoolName(PoolName PoolName); [Fact] public void PoolNameDeserializesFromString() { diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index efbf12dc9d..63f9c493f9 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -108,9 +108,9 @@ version = "0.1.53" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" dependencies = [ - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", - "syn 1.0.95", + "syn 1.0.103", ] [[package]] @@ -238,7 +238,7 @@ dependencies = [ "cc", "cfg-if 1.0.0", "libc", - "miniz_oxide", + "miniz_oxide 0.4.4", "object", "rustc-demangle", ] @@ -249,6 +249,12 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "binary-merge" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597bb81c80a54b6a4381b23faba8d7774b144c94cbd1d6fe3f1329bd776554ab" + [[package]] name = "bincode" version = "1.3.3" @@ -273,7 +279,7 @@ dependencies = [ "lazycell", "log", "peeking_take_while", - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", "regex", "rustc-hash", @@ -329,6 +335,15 @@ dependencies = [ "arrayvec 0.7.2", ] +[[package]] +name = "brownstone" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5839ee4f953e811bfdcf223f509cb2c6a3e1447959b0bff459405575bc17f22" +dependencies = [ + "arrayvec 0.7.2", +] + [[package]] name = "bumpalo" version = "3.7.1" @@ -436,8 +451,8 @@ checksum = "6d20de3739b4fb45a17837824f40aa1769cc7655d7a83e68739a77fe7b30c87a" dependencies = [ "atty", "bitflags", - "clap_derive", - "clap_lex", + "clap_derive 3.2.4", + "clap_lex 0.2.2", "indexmap", "once_cell", "strsim 0.10.0", @@ -445,6 +460,21 @@ dependencies = [ "textwrap 0.15.0", ] +[[package]] +name = "clap" +version = "4.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2148adefda54e14492fb9bddcc600b4344c5d1a3123bd666dcb939c6f0e0e57e" +dependencies = [ + "atty", + "bitflags", + "clap_derive 4.0.21", + "clap_lex 0.3.0", + "once_cell", + "strsim 0.10.0", + "termcolor", +] + [[package]] name = "clap_derive" version = "3.2.4" @@ -453,9 +483,22 @@ checksum = "026baf08b89ffbd332836002ec9378ef0e69648cbfadd68af7cd398ca5bf98f7" dependencies = [ "heck 0.4.0", "proc-macro-error", - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", + "quote 1.0.9", + "syn 1.0.103", +] + +[[package]] +name = "clap_derive" +version = "4.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" +dependencies = [ + "heck 0.4.0", + "proc-macro-error", + "proc-macro2 1.0.47", "quote 1.0.9", - "syn 1.0.95", + "syn 1.0.103", ] [[package]] @@ -467,6 +510,15 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "clap_lex" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "concurrent-queue" version = "1.2.2" @@ -503,18 +555,35 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "coverage" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.0.26", + "debuggable-module", + "debugger", + "iced-x86", + "log", + "pete", + "procfs", + "regex", + "symbolic 10.1.4", + "thiserror", +] + [[package]] name = "coverage-legacy" version = "0.1.0" dependencies = [ "anyhow", "bincode", - "cpp_demangle", + "cpp_demangle 0.3.5", "debugger", "dunce", "env_logger 0.9.0", "fixedbitset", - "goblin", + "goblin 0.5.1", "iced-x86", "log", "memmap2", @@ -529,7 +598,7 @@ dependencies = [ "serde", "serde_json", "structopt", - "symbolic", + "symbolic 8.8.0", "uuid 0.8.2", "win-util", "winapi", @@ -544,6 +613,15 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "cpp_demangle" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b446fd40bcc17eddd6a4a78f24315eb90afdb3334999ddfd4909985c47722442" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "cpufeatures" version = "0.2.5" @@ -555,9 +633,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.2.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ "cfg-if 1.0.0", ] @@ -608,12 +686,11 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.7" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e5bed1f1c269533fa816a0a5492b3545209a205ca1a54842be180eb63a16a6" +checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" dependencies = [ "cfg-if 1.0.0", - "lazy_static", ] [[package]] @@ -674,7 +751,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccc0a48a9b826acdf4028595adc9db92caea352f7af011a3034acd172a52a0aa" dependencies = [ "quote 1.0.9", - "syn 1.0.95", + "syn 1.0.103", ] [[package]] @@ -701,13 +778,29 @@ dependencies = [ "tokio", ] +[[package]] +name = "debuggable-module" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.0.26", + "gimli", + "goblin 0.6.0", + "iced-x86", + "log", + "pdb 0.8.0", + "regex", + "symbolic 10.1.4", + "thiserror", +] + [[package]] name = "debugger" version = "0.1.0" dependencies = [ "anyhow", "fnv", - "goblin", + "goblin 0.5.1", "iced-x86", "log", "memmap2", @@ -726,15 +819,24 @@ dependencies = [ "uuid 0.8.2", ] +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "uuid 1.2.1", +] + [[package]] name = "derivative" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", - "syn 1.0.95", + "syn 1.0.103", ] [[package]] @@ -814,6 +916,24 @@ dependencies = [ "xml-rs", ] +[[package]] +name = "elementtree" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e91f812124e4fc7f82605feda4da357307185a4619baee415ae0b7f6d5e031" +dependencies = [ + "string_cache", +] + +[[package]] +name = "elsa" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b4b5d23ed6b6948d68240aafa4ac98e568c9a020efd9d4201a6288bc3006e09" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "encoding_rs" version = "0.8.28" @@ -911,14 +1031,12 @@ checksum = "398ea4fabe40b9b0d885340a2a991a44c8a645624075ad966d21f88688e2b69e" [[package]] name = "flate2" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" dependencies = [ - "cfg-if 1.0.0", "crc32fast", - "libc", - "miniz_oxide", + "miniz_oxide 0.5.4", ] [[package]] @@ -965,9 +1083,9 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63f713f8b2aa9e24fec85b0e290c56caee12e3b6ae0aeeda238a75b28251afd6" dependencies = [ - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", - "syn 1.0.95", + "syn 1.0.103", ] [[package]] @@ -1082,9 +1200,9 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" dependencies = [ - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", - "syn 1.0.95", + "syn 1.0.103", ] [[package]] @@ -1165,11 +1283,12 @@ dependencies = [ [[package]] name = "gimli" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" +checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" dependencies = [ "fallible-iterator", + "indexmap", "stable_deref_trait", ] @@ -1190,6 +1309,17 @@ dependencies = [ "scroll 0.11.0", ] +[[package]] +name = "goblin" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "572564d6cba7d09775202c8e7eebc4d534d5ae36578ab402fb21e182a0ac9505" +dependencies = [ + "log", + "plain", + "scroll 0.11.0", +] + [[package]] name = "h2" version = "0.3.13" @@ -1211,9 +1341,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.11.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "heck" @@ -1392,9 +1522,9 @@ checksum = "0cfe9645a18782869361d9c8732246be7b410ad4e919d3609ebabdac00ba12c3" [[package]] name = "indexmap" -version = "1.7.0" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", "hashbrown", @@ -1426,6 +1556,15 @@ dependencies = [ "libc", ] +[[package]] +name = "inplace-vec-builder" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf64c2edc8226891a71f127587a2861b132d2b942310843814d5001d99a1d307" +dependencies = [ + "smallvec", +] + [[package]] name = "input-tester" version = "0.1.0" @@ -1517,6 +1656,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + [[package]] name = "lexical-core" version = "0.7.6" @@ -1577,9 +1722,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.14" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if 1.0.0", ] @@ -1605,6 +1750,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +[[package]] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" + [[package]] name = "md5" version = "0.7.0" @@ -1657,6 +1808,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "miniz_oxide" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.7.13" @@ -1789,7 +1949,20 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aadc66631948f6b65da03be4c4cd8bd104d481697ecbb9bbd65719b1ec60bc9f" dependencies = [ - "brownstone", + "brownstone 1.1.0", + "indent_write", + "joinery", + "memchr", + "nom 7.1.0", +] + +[[package]] +name = "nom-supreme" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bd3ae6c901f1959588759ff51c95d24b491ecb9ff91aa9c2ef4acc5b1dcab27" +dependencies = [ + "brownstone 3.0.0", "indent_write", "joinery", "memchr", @@ -1872,9 +2045,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.12.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" +checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" [[package]] name = "onefuzz" @@ -1885,7 +2058,7 @@ dependencies = [ "backoff", "base64", "bytes", - "cpp_demangle", + "cpp_demangle 0.3.5", "debugger", "dunce", "dynamic-library", @@ -2038,9 +2211,9 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", - "syn 1.0.95", + "syn 1.0.103", ] [[package]] @@ -2197,6 +2370,20 @@ dependencies = [ "uuid 1.2.1", ] +[[package]] +name = "pdb-addr2line" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4e89a9f2f40b2389ba6da0814c8044bf942bece03dffa1514f84e3b525f4f9a" +dependencies = [ + "bitflags", + "elsa", + "maybe-owned", + "pdb 0.8.0", + "range-collections", + "thiserror", +] + [[package]] name = "peeking_take_while" version = "0.1.2" @@ -2245,16 +2432,16 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389" dependencies = [ - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", - "syn 1.0.95", + "syn 1.0.103", ] [[package]] name = "pin-project-lite" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pin-utils" @@ -2305,9 +2492,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", - "syn 1.0.95", + "syn 1.0.103", "version_check", ] @@ -2317,7 +2504,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", "version_check", ] @@ -2333,9 +2520,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.39" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" dependencies = [ "unicode-ident", ] @@ -2413,7 +2600,7 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" dependencies = [ - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", ] [[package]] @@ -2509,6 +2696,17 @@ dependencies = [ "rand_core 0.6.3", ] +[[package]] +name = "range-collections" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61fdfd79629e2b44a1d34b4d227957174cb858e6b86ee45fad114edbcfc903ab" +dependencies = [ + "binary-merge", + "inplace-vec-builder", + "smallvec", +] + [[package]] name = "rayon" version = "1.5.1" @@ -2712,9 +2910,9 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdbda6ac5cd1321e724fa9cee216f3a61885889b896f073b8f82322789c5250e" dependencies = [ - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", - "syn 1.0.95", + "syn 1.0.103", ] [[package]] @@ -2773,9 +2971,9 @@ version = "1.0.136" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" dependencies = [ - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", - "syn 1.0.95", + "syn 1.0.103", ] [[package]] @@ -2894,9 +3092,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.6.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "snafu" @@ -2915,9 +3113,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "410b26ed97440d90ced3e2488c868d56a86e2064f5d7d6f417909b286afe25e5" dependencies = [ "heck 0.4.0", - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", - "syn 1.0.95", + "syn 1.0.103", ] [[package]] @@ -3050,9 +3248,9 @@ checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" dependencies = [ "heck 0.3.3", "proc-macro-error", - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", - "syn 1.0.95", + "syn 1.0.103", ] [[package]] @@ -3068,10 +3266,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6878079b17446e4d3eba6192bb0a2950d5b14f0ed8424b852310e5a94345d0ef" dependencies = [ "heck 0.4.0", - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", "rustversion", - "syn 1.0.95", + "syn 1.0.103", ] [[package]] @@ -3086,10 +3284,22 @@ version = "8.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b49345d083b1103e25c8c10e5e52cff254d33e70e29307c2bc4777074a25258" dependencies = [ - "symbolic-common", - "symbolic-debuginfo", - "symbolic-demangle", - "symbolic-symcache", + "symbolic-common 8.8.0", + "symbolic-debuginfo 8.8.0", + "symbolic-demangle 8.8.0", + "symbolic-symcache 8.8.0", +] + +[[package]] +name = "symbolic" +version = "10.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b13e4f7150867965f7e7f29c4e6632fbdf1f2d028812012c930b970d1e61ac0" +dependencies = [ + "symbolic-common 10.1.4", + "symbolic-debuginfo 10.1.4", + "symbolic-demangle 10.1.4", + "symbolic-symcache 10.1.4", ] [[package]] @@ -3098,12 +3308,24 @@ version = "8.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f551f902d5642e58039aee6a9021a61037926af96e071816361644983966f540" dependencies = [ - "debugid", + "debugid 0.7.2", "memmap2", "stable_deref_trait", "uuid 0.8.2", ] +[[package]] +name = "symbolic-common" +version = "10.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "878e1296b4dbe9f77102748d66abc25f6089a020f03a4bba8530b48221e131f3" +dependencies = [ + "debugid 0.8.0", + "memmap2", + "stable_deref_trait", + "uuid 1.2.1", +] + [[package]] name = "symbolic-debuginfo" version = "8.8.0" @@ -3112,15 +3334,15 @@ checksum = "1165dabf9fc1d6bb6819c2c0e27c8dd0e3068d2c53cf186d319788e96517f0d6" dependencies = [ "bitvec 1.0.0", "dmsort", - "elementtree", + "elementtree 0.7.0", "fallible-iterator", "flate2", "gimli", - "goblin", + "goblin 0.5.1", "lazy_static", "lazycell", "nom 7.1.0", - "nom-supreme", + "nom-supreme 0.6.0", "parking_lot 0.12.1", "pdb 0.7.0", "regex", @@ -3128,10 +3350,42 @@ dependencies = [ "serde", "serde_json", "smallvec", - "symbolic-common", + "symbolic-common 8.8.0", "thiserror", - "wasmparser", - "zip", + "wasmparser 0.83.0", + "zip 0.5.13", +] + +[[package]] +name = "symbolic-debuginfo" +version = "10.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f609bccfbb7a009b8e20ea388f9169716721f1e9d375b9c2cd6710c41a3a590" +dependencies = [ + "bitvec 1.0.0", + "dmsort", + "elementtree 1.2.2", + "elsa", + "fallible-iterator", + "flate2", + "gimli", + "goblin 0.6.0", + "lazy_static", + "lazycell", + "nom 7.1.0", + "nom-supreme 0.8.0", + "parking_lot 0.12.1", + "pdb-addr2line", + "regex", + "scroll 0.11.0", + "serde", + "serde_json", + "smallvec", + "symbolic-common 10.1.4", + "symbolic-ppdb", + "thiserror", + "wasmparser 0.94.0", + "zip 0.6.3", ] [[package]] @@ -3141,10 +3395,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4564ca7b4e6eb14105aa8bbbce26e080f6b5d9c4373e67167ab31f7b86443750" dependencies = [ "cc", - "cpp_demangle", + "cpp_demangle 0.3.5", + "msvc-demangler", + "rustc-demangle", + "symbolic-common 8.8.0", +] + +[[package]] +name = "symbolic-demangle" +version = "10.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cb721346d1f73b20c50b3122f1b1b519465894b9378e47762623dc05e44b5b3" +dependencies = [ + "cc", + "cpp_demangle 0.4.0", "msvc-demangler", "rustc-demangle", - "symbolic-common", + "symbolic-common 10.1.4", +] + +[[package]] +name = "symbolic-ppdb" +version = "10.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b929ab808f03b65d46c2f65e0b21cc24eefc5b2357f93442be6ef9170478e490" +dependencies = [ + "indexmap", + "symbolic-common 10.1.4", + "thiserror", + "uuid 1.2.1", + "watto", ] [[package]] @@ -3156,11 +3436,25 @@ dependencies = [ "dmsort", "fnv", "indexmap", - "symbolic-common", - "symbolic-debuginfo", + "symbolic-common 8.8.0", + "symbolic-debuginfo 8.8.0", "thiserror", ] +[[package]] +name = "symbolic-symcache" +version = "10.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31608caf5e6797dafdcda5fe835122fb821fb3b14c0eaa472ee5979d6d2835e2" +dependencies = [ + "indexmap", + "symbolic-common 10.1.4", + "symbolic-debuginfo 10.1.4", + "thiserror", + "tracing", + "watto", +] + [[package]] name = "syn" version = "0.15.44" @@ -3174,11 +3468,11 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.95" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbaf6116ab8924f39d52792136fb74fd60a80194cf1b1c6ffa6453eef1c3f942" +checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" dependencies = [ - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", "unicode-ident", ] @@ -3229,22 +3523,22 @@ checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" [[package]] name = "thiserror" -version = "1.0.30" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.30" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" dependencies = [ - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", - "syn 1.0.95", + "syn 1.0.103", ] [[package]] @@ -3306,9 +3600,9 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" dependencies = [ - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", - "syn 1.0.95", + "syn 1.0.103", ] [[package]] @@ -3372,9 +3666,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.28" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84f96e095c0c82419687c20ddf5cb3eadb61f4e1405923c9dc8e53a1adacbda8" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if 1.0.0", "pin-project-lite", @@ -3384,22 +3678,22 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.21" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", - "syn 1.0.95", + "syn 1.0.103", ] [[package]] name = "tracing-core" -version = "0.1.20" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46125608c26121c81b0c6d693eab5a420e416da7e43c426d2e8f7df8da8a3acf" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ - "lazy_static", + "once_cell", ] [[package]] @@ -3626,9 +3920,9 @@ dependencies = [ "bumpalo", "lazy_static", "log", - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", - "syn 1.0.95", + "syn 1.0.103", "wasm-bindgen-shared", ] @@ -3660,9 +3954,9 @@ version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab" dependencies = [ - "proc-macro2 1.0.39", + "proc-macro2 1.0.47", "quote 1.0.9", - "syn 1.0.95", + "syn 1.0.103", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3679,6 +3973,25 @@ version = "0.83.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "718ed7c55c2add6548cca3ddd6383d738cd73b892df400e96b9aa876f0141d7a" +[[package]] +name = "wasmparser" +version = "0.94.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdac7e1d98d70913ae3b4923dd7419c8ea7bdfd4c44a240a0ba305d929b7f191" +dependencies = [ + "indexmap", +] + +[[package]] +name = "watto" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6746b5315e417144282a047ebb82260d45c92d09bf653fa9ec975e3809be942b" +dependencies = [ + "leb128", + "thiserror", +] + [[package]] name = "web-sys" version = "0.3.55" @@ -3866,3 +4179,15 @@ dependencies = [ "flate2", "thiserror", ] + +[[package]] +name = "zip" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537ce7411d25e54e8ae21a7ce0b15840e7bfcff15b51d697ec3266cc76bdf080" +dependencies = [ + "byteorder", + "crc32fast", + "crossbeam-utils", + "flate2", +] diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index 74a5d4f3d5..2421c248e6 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -1,7 +1,9 @@ [workspace] members = [ "atexit", + "coverage", "coverage-legacy", + "debuggable-module", "debugger", "dynamic-library", "input-tester", diff --git a/src/agent/coverage/Cargo.toml b/src/agent/coverage/Cargo.toml new file mode 100644 index 0000000000..00bbfbf4c8 --- /dev/null +++ b/src/agent/coverage/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "coverage" +version = "0.1.0" +edition = "2021" +license = "MIT" + +[dependencies] +anyhow = "1.0" +debuggable-module = { path = "../debuggable-module" } +iced-x86 = "1.17" +log = "0.4.17" +regex = "1.0" +symbolic = { version = "10.1", features = ["debuginfo", "demangle", "symcache"] } +thiserror = "1.0" + +[target.'cfg(target_os = "windows")'.dependencies] +debugger = { path = "../debugger" } + +[target.'cfg(target_os = "linux")'.dependencies] +pete = "0.9" +# For procfs, opt out of the `chrono` freature; it pulls in an old version +# of `time`. We do not use the methods that the `chrono` feature enables. +procfs = { version = "0.12", default-features = false, features=["flate2"] } + +[dev-dependencies] +clap = { version = "4.0", features = ["derive"] } diff --git a/src/agent/coverage/examples/coverage.rs b/src/agent/coverage/examples/coverage.rs new file mode 100644 index 0000000000..e9e0e5b1da --- /dev/null +++ b/src/agent/coverage/examples/coverage.rs @@ -0,0 +1,65 @@ +use std::process::Command; +use std::time::Duration; + +use anyhow::Result; +use clap::Parser; +use coverage::allowlist::{AllowList, TargetAllowList}; +use coverage::binary::BinaryCoverage; + +#[derive(Parser, Debug)] +struct Args { + #[arg(long)] + module_allowlist: Option, + + #[arg(long)] + source_allowlist: Option, + + #[arg(short, long)] + timeout: Option, + + command: Vec, +} + +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5); + +fn main() -> Result<()> { + let args = Args::parse(); + + let timeout = args + .timeout + .map(Duration::from_millis) + .unwrap_or(DEFAULT_TIMEOUT); + + let mut cmd = Command::new(&args.command[0]); + if args.command.len() > 1 { + cmd.args(&args.command[1..]); + } + + let mut allowlist = TargetAllowList::default(); + + if let Some(path) = &args.module_allowlist { + allowlist.modules = AllowList::load(path)?; + } + + if let Some(path) = &args.source_allowlist { + allowlist.source_files = AllowList::load(path)?; + } + + let coverage = coverage::record::record(cmd, timeout, allowlist)?; + + dump_modoff(coverage)?; + + Ok(()) +} + +fn dump_modoff(coverage: BinaryCoverage) -> Result<()> { + for (module, coverage) in &coverage.modules { + for (offset, count) in coverage.as_ref() { + if count.reached() { + println!("{}+{offset:x}", module.base_name()); + } + } + } + + Ok(()) +} diff --git a/src/agent/coverage/src/allowlist.rs b/src/agent/coverage/src/allowlist.rs new file mode 100644 index 0000000000..2a0e807d17 --- /dev/null +++ b/src/agent/coverage/src/allowlist.rs @@ -0,0 +1,156 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use anyhow::Result; +use regex::{Regex, RegexSet}; +use std::path::Path; + +#[derive(Clone, Debug, Default)] +pub struct TargetAllowList { + pub functions: AllowList, + pub modules: AllowList, + pub source_files: AllowList, +} + +impl TargetAllowList { + pub fn new(modules: AllowList, source_files: AllowList) -> Self { + // Allow all. + let functions = AllowList::default(); + + Self { + functions, + modules, + source_files, + } + } +} + +#[derive(Clone, Debug)] +pub struct AllowList { + allow: RegexSet, + deny: RegexSet, +} + +impl AllowList { + pub fn new(allow: RegexSet, deny: RegexSet) -> Self { + Self { allow, deny } + } + + pub fn load(path: impl AsRef) -> Result { + let path = path.as_ref(); + let text = std::fs::read_to_string(path)?; + Self::parse(&text) + } + + pub fn parse(text: &str) -> Result { + use std::io::{BufRead, BufReader}; + + let reader = BufReader::new(text.as_bytes()); + + let mut allow = vec![]; + let mut deny = vec![]; + + // We could just collect and pass to the `RegexSet` ctor. + // + // Instead, check each rule individually for diagnostic purposes. + for (index, line) in reader.lines().enumerate() { + let line = line?; + + match AllowListLine::parse(&line) { + Ok(valid) => { + use AllowListLine::*; + + match valid { + Blank | Comment => { + // Ignore. + } + Allow(re) => { + allow.push(re); + } + Deny(re) => { + deny.push(re); + } + } + } + Err(err) => { + // Ignore invalid lines, but warn. + let line_number = index + 1; + warn!("error at line {}: {}", line_number, err); + } + } + } + + let allow = RegexSet::new(allow.iter().map(|re| re.as_str()))?; + let deny = RegexSet::new(deny.iter().map(|re| re.as_str()))?; + let allowlist = AllowList::new(allow, deny); + + Ok(allowlist) + } + + pub fn is_allowed(&self, path: impl AsRef) -> bool { + let path = path.as_ref(); + + // Allowed if rule-allowed but not excluded by a negative (deny) rule. + self.allow.is_match(path) && !self.deny.is_match(path) + } +} + +impl Default for AllowList { + fn default() -> Self { + // Unwrap-safe due to valid constant expr. + let allow = RegexSet::new([".*"]).unwrap(); + let deny = RegexSet::empty(); + + AllowList::new(allow, deny) + } +} + +pub enum AllowListLine { + Blank, + Comment, + Allow(Regex), + Deny(Regex), +} + +impl AllowListLine { + pub fn parse(line: &str) -> Result { + let line = line.trim(); + + // Allow and ignore blank lines. + if line.is_empty() { + return Ok(Self::Blank); + } + + // Support comments of the form `# `. + if line.starts_with("# ") { + return Ok(Self::Comment); + } + + // Deny rules are of the form `! `. + if let Some(expr) = line.strip_prefix("! ") { + let re = glob_to_regex(expr)?; + return Ok(Self::Deny(re)); + } + + // Try to interpret as allow rule. + let re = glob_to_regex(line)?; + Ok(Self::Allow(re)) + } +} + +#[allow(clippy::single_char_pattern)] +fn glob_to_regex(expr: &str) -> Result { + // Don't make users escape Windows path separators. + let expr = expr.replace(r"\", r"\\"); + + // Translate glob wildcards into quantified regexes. + let expr = expr.replace("*", ".*"); + + // Anchor to line start and end. + let expr = format!("^{expr}$"); + + Ok(Regex::new(&expr)?) +} + +#[cfg(test)] +mod tests; diff --git a/src/agent/coverage/src/allowlist/test-data/allow-all-glob-except-commented.txt b/src/agent/coverage/src/allowlist/test-data/allow-all-glob-except-commented.txt new file mode 100644 index 0000000000..6ef5c08319 --- /dev/null +++ b/src/agent/coverage/src/allowlist/test-data/allow-all-glob-except-commented.txt @@ -0,0 +1,3 @@ +a/* +! a/c +# c diff --git a/src/agent/coverage/src/allowlist/test-data/allow-all-glob-except.txt b/src/agent/coverage/src/allowlist/test-data/allow-all-glob-except.txt new file mode 100644 index 0000000000..a028542f1e --- /dev/null +++ b/src/agent/coverage/src/allowlist/test-data/allow-all-glob-except.txt @@ -0,0 +1,3 @@ +a/* +! a/c +c diff --git a/src/agent/coverage/src/allowlist/test-data/allow-all-glob.txt b/src/agent/coverage/src/allowlist/test-data/allow-all-glob.txt new file mode 100644 index 0000000000..72e8ffc0db --- /dev/null +++ b/src/agent/coverage/src/allowlist/test-data/allow-all-glob.txt @@ -0,0 +1 @@ +* diff --git a/src/agent/coverage/src/allowlist/test-data/allow-all.txt b/src/agent/coverage/src/allowlist/test-data/allow-all.txt new file mode 100644 index 0000000000..0ee448f0b5 --- /dev/null +++ b/src/agent/coverage/src/allowlist/test-data/allow-all.txt @@ -0,0 +1,4 @@ +a +a/b +b +c diff --git a/src/agent/coverage/src/allowlist/test-data/allow-some.txt b/src/agent/coverage/src/allowlist/test-data/allow-some.txt new file mode 100644 index 0000000000..422c2b7ab3 --- /dev/null +++ b/src/agent/coverage/src/allowlist/test-data/allow-some.txt @@ -0,0 +1,2 @@ +a +b diff --git a/src/agent/coverage/src/allowlist/test-data/empty.txt b/src/agent/coverage/src/allowlist/test-data/empty.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/agent/coverage/src/allowlist/tests.rs b/src/agent/coverage/src/allowlist/tests.rs new file mode 100644 index 0000000000..7c189aae88 --- /dev/null +++ b/src/agent/coverage/src/allowlist/tests.rs @@ -0,0 +1,101 @@ +use anyhow::Result; + +use super::AllowList; + +#[test] +fn test_default() -> Result<()> { + let allowlist = AllowList::default(); + + // All allowed. + assert!(allowlist.is_allowed("a")); + assert!(allowlist.is_allowed("a/b")); + assert!(allowlist.is_allowed("b")); + assert!(allowlist.is_allowed("c")); + + Ok(()) +} + +#[test] +fn test_empty() -> Result<()> { + let text = include_str!("test-data/empty.txt"); + let allowlist = AllowList::parse(text)?; + + // All excluded. + assert!(!allowlist.is_allowed("a")); + assert!(!allowlist.is_allowed("a/b")); + assert!(!allowlist.is_allowed("b")); + assert!(!allowlist.is_allowed("c")); + + Ok(()) +} + +#[test] +fn test_allow_some() -> Result<()> { + let text = include_str!("test-data/allow-some.txt"); + let allowlist = AllowList::parse(text)?; + + assert!(allowlist.is_allowed("a")); + assert!(!allowlist.is_allowed("a/b")); + assert!(allowlist.is_allowed("b")); + assert!(!allowlist.is_allowed("c")); + + Ok(()) +} + +#[test] +fn test_allow_all() -> Result<()> { + let text = include_str!("test-data/allow-all.txt"); + let allowlist = AllowList::parse(text)?; + + assert!(allowlist.is_allowed("a")); + assert!(allowlist.is_allowed("a/b")); + assert!(allowlist.is_allowed("b")); + assert!(allowlist.is_allowed("c")); + + Ok(()) +} + +#[test] +fn test_allow_all_glob() -> Result<()> { + let text = include_str!("test-data/allow-all-glob.txt"); + let allowlist = AllowList::parse(text)?; + + assert!(allowlist.is_allowed("a")); + assert!(allowlist.is_allowed("a/b")); + assert!(allowlist.is_allowed("b")); + assert!(allowlist.is_allowed("c")); + + Ok(()) +} + +#[test] +fn test_allow_glob_except() -> Result<()> { + let text = include_str!("test-data/allow-all-glob-except.txt"); + let allowlist = AllowList::parse(text)?; + + assert!(!allowlist.is_allowed("a")); + assert!(allowlist.is_allowed("a/b")); + assert!(!allowlist.is_allowed("a/c")); + assert!(allowlist.is_allowed("a/d")); + assert!(!allowlist.is_allowed("b")); + assert!(allowlist.is_allowed("c")); + + Ok(()) +} + +#[test] +fn test_allow_glob_except_commented() -> Result<()> { + let text = include_str!("test-data/allow-all-glob-except-commented.txt"); + let allowlist = AllowList::parse(text)?; + + assert!(!allowlist.is_allowed("a")); + assert!(allowlist.is_allowed("a/b")); + assert!(!allowlist.is_allowed("a/c")); + assert!(allowlist.is_allowed("a/d")); + assert!(!allowlist.is_allowed("b")); + + // Allowed by the rule `c`, but not allowed because `# c` is a comment. + assert!(!allowlist.is_allowed("c")); + + Ok(()) +} diff --git a/src/agent/coverage/src/binary.rs b/src/agent/coverage/src/binary.rs new file mode 100644 index 0000000000..0ef6640f47 --- /dev/null +++ b/src/agent/coverage/src/binary.rs @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::collections::{BTreeMap, BTreeSet}; + +use anyhow::{bail, Result}; +use debuggable_module::{block, path::FilePath, Module, Offset}; +use symbolic::debuginfo::Object; +use symbolic::symcache::{SymCache, SymCacheConverter}; + +use crate::allowlist::TargetAllowList; + +#[derive(Clone, Debug, Default)] +pub struct BinaryCoverage { + pub modules: BTreeMap, +} + +#[derive(Clone, Debug, Default)] +pub struct ModuleBinaryCoverage { + pub offsets: BTreeMap, +} + +impl ModuleBinaryCoverage { + pub fn increment(&mut self, offset: Offset) -> Result<()> { + if let Some(count) = self.offsets.get_mut(&offset) { + count.increment(); + } else { + bail!("unknown coverage offset: {offset:x}"); + }; + + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct Count(pub u32); + +impl Count { + pub fn increment(&mut self) { + self.0 = self.0.saturating_add(1); + } + + pub fn reached(&self) -> bool { + self.0 > 0 + } +} + +pub fn find_coverage_sites<'data>( + module: &dyn Module<'data>, + allowlist: &TargetAllowList, +) -> Result { + let debuginfo = module.debuginfo()?; + + let mut symcache = vec![]; + let mut converter = SymCacheConverter::new(); + let exe = Object::parse(module.executable_data())?; + converter.process_object(&exe)?; + let di = Object::parse(module.debuginfo_data())?; + converter.process_object(&di)?; + converter.serialize(&mut std::io::Cursor::new(&mut symcache))?; + let symcache = SymCache::parse(&symcache)?; + + let mut offsets = BTreeSet::new(); + + for function in debuginfo.functions() { + if !allowlist.functions.is_allowed(&function.name) { + continue; + } + + if let Some(location) = symcache.lookup(function.offset.0).next() { + if let Some(file) = location.file() { + let path = file.full_path(); + + if allowlist.source_files.is_allowed(&path) { + let blocks = + block::sweep_region(module, &debuginfo, function.offset, function.size)?; + offsets.extend(blocks.iter().map(|b| b.offset)); + } + } + } + } + + let mut coverage = ModuleBinaryCoverage::default(); + coverage + .offsets + .extend(offsets.into_iter().map(|o| (o, Count(0)))); + + Ok(coverage) +} + +impl AsRef> for ModuleBinaryCoverage { + fn as_ref(&self) -> &BTreeMap { + &self.offsets + } +} diff --git a/src/agent/coverage/src/lib.rs b/src/agent/coverage/src/lib.rs new file mode 100644 index 0000000000..fe395cdf4a --- /dev/null +++ b/src/agent/coverage/src/lib.rs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#[macro_use] +extern crate log; + +pub mod allowlist; +pub mod binary; +pub mod record; +pub mod source; +mod timer; + +#[doc(inline)] +pub use allowlist::{AllowList, TargetAllowList}; + +#[doc(inline)] +pub use record::record; diff --git a/src/agent/coverage/src/record.rs b/src/agent/coverage/src/record.rs new file mode 100644 index 0000000000..92336d0065 --- /dev/null +++ b/src/agent/coverage/src/record.rs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#[cfg(target_os = "linux")] +pub mod linux; + +#[cfg(target_os = "windows")] +pub mod windows; + +#[cfg(target_os = "linux")] +pub use crate::record::linux::record; + +#[cfg(target_os = "windows")] +pub use crate::record::windows::record; diff --git a/src/agent/coverage/src/record/linux.rs b/src/agent/coverage/src/record/linux.rs new file mode 100644 index 0000000000..4f331a3241 --- /dev/null +++ b/src/agent/coverage/src/record/linux.rs @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::collections::BTreeMap; +use std::process::Command; +use std::time::Duration; + +use anyhow::{bail, Result}; +use debuggable_module::linux::LinuxModule; +use debuggable_module::load_module::LoadModule; +use debuggable_module::loader::Loader; +use debuggable_module::path::FilePath; +use debuggable_module::Address; +use pete::Tracee; + +pub mod debugger; +use debugger::{DebugEventHandler, Debugger, DebuggerContext, ModuleImage}; + +use crate::allowlist::TargetAllowList; +use crate::binary::{self, BinaryCoverage}; + +pub fn record( + cmd: Command, + timeout: Duration, + allowlist: impl Into>, +) -> Result { + let loader = Loader::new(); + let allowlist = allowlist.into().unwrap_or_default(); + + crate::timer::timed(timeout, move || { + let mut recorder = LinuxRecorder::new(&loader, allowlist); + let dbg = Debugger::new(&mut recorder); + dbg.run(cmd)?; + + Ok(recorder.coverage) + })? +} + +pub struct LinuxRecorder<'data> { + allowlist: TargetAllowList, + coverage: BinaryCoverage, + loader: &'data Loader, + modules: BTreeMap>, +} + +impl<'data> LinuxRecorder<'data> { + pub fn new(loader: &'data Loader, allowlist: TargetAllowList) -> Self { + let coverage = BinaryCoverage::default(); + let modules = BTreeMap::new(); + + Self { + allowlist, + coverage, + loader, + modules, + } + } + + fn do_on_breakpoint( + &mut self, + context: &mut DebuggerContext, + tracee: &mut Tracee, + ) -> Result<()> { + let regs = tracee.registers()?; + let addr = Address(regs.rip); + + if let Some(image) = context.find_image_for_addr(addr) { + if let Some(coverage) = self.coverage.modules.get_mut(image.path()) { + let offset = addr.offset_from(image.base())?; + coverage.increment(offset)?; + } else { + bail!("coverage not initialized for module {}", image.path()); + } + } else { + bail!("no image for addr: {addr:x}"); + } + + Ok(()) + } + + fn do_on_module_load( + &mut self, + context: &mut DebuggerContext, + tracee: &mut Tracee, + image: &ModuleImage, + ) -> Result<()> { + info!("module load: {}", image.path()); + + let path = image.path(); + + if !self.allowlist.modules.is_allowed(path) { + debug!("not inserting denylisted module: {path}"); + return Ok(()); + } + + let module = if let Ok(module) = LinuxModule::load(self.loader, path.clone()) { + module + } else { + debug!("skipping undebuggable module: {path}"); + return Ok(()); + }; + + let coverage = binary::find_coverage_sites(&module, &self.allowlist)?; + + for offset in coverage.as_ref().keys().copied() { + let addr = image.base().offset_by(offset)?; + context.breakpoints.set(tracee, addr)?; + } + + self.coverage.modules.insert(path.clone(), coverage); + + self.modules.insert(path.clone(), module); + + Ok(()) + } +} + +impl<'data> DebugEventHandler for LinuxRecorder<'data> { + fn on_breakpoint(&mut self, context: &mut DebuggerContext, tracee: &mut Tracee) -> Result<()> { + self.do_on_breakpoint(context, tracee) + } + + fn on_module_load( + &mut self, + context: &mut DebuggerContext, + tracee: &mut Tracee, + image: &ModuleImage, + ) -> Result<()> { + self.do_on_module_load(context, tracee, image) + } +} diff --git a/src/agent/coverage/src/record/linux/debugger.rs b/src/agent/coverage/src/record/linux/debugger.rs new file mode 100644 index 0000000000..5f300d6ccc --- /dev/null +++ b/src/agent/coverage/src/record/linux/debugger.rs @@ -0,0 +1,370 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::collections::BTreeMap; +use std::process::Command; + +use anyhow::{bail, format_err, Result}; +use debuggable_module::path::FilePath; +use debuggable_module::Address; +use pete::{Ptracer, Restart, Signal, Stop, Tracee}; +use procfs::process::{MMapPath, MemoryMap, Process}; + +pub trait DebugEventHandler { + fn on_breakpoint(&mut self, dbg: &mut DebuggerContext, tracee: &mut Tracee) -> Result<()>; + + fn on_module_load( + &mut self, + db: &mut DebuggerContext, + tracee: &mut Tracee, + image: &ModuleImage, + ) -> Result<()>; +} + +pub struct Debugger<'eh> { + context: DebuggerContext, + event_handler: &'eh mut dyn DebugEventHandler, +} + +impl<'eh> Debugger<'eh> { + pub fn new(event_handler: &'eh mut dyn DebugEventHandler) -> Self { + let context = DebuggerContext::new(); + + Self { + context, + event_handler, + } + } + + pub fn run(mut self, cmd: Command) -> Result<()> { + let mut child = self.context.tracer.spawn(cmd)?; + + if let Err(err) = self.wait_on_stops() { + // Ignore error if child already exited. + let _ = child.kill(); + + return Err(err); + } + + Ok(()) + } + + fn wait_on_stops(mut self) -> Result<()> { + use pete::ptracer::Options; + + // Continue the tracee process until the return from its initial `execve()`. + let mut tracee = continue_to_init_execve(&mut self.context.tracer)?; + + // Do not follow forks. + // + // After this, we assume that any new tracee is a thread in the same + // group as the root tracee. + let mut options = Options::all(); + options.remove(Options::PTRACE_O_TRACEFORK); + options.remove(Options::PTRACE_O_TRACEVFORK); + options.remove(Options::PTRACE_O_TRACEEXEC); + tracee.set_options(options)?; + + // Initialize index of mapped modules now that we have a PID to query. + self.context.images = Some(Images::new(tracee.pid.as_raw())); + self.update_images(&mut tracee)?; + + // Restart tracee and enter the main debugger loop. + self.context.tracer.restart(tracee, Restart::Syscall)?; + + while let Some(mut tracee) = self.context.tracer.wait()? { + match tracee.stop { + Stop::SyscallEnter => trace!("syscall-enter: {:?}", tracee.stop), + Stop::SyscallExit => { + self.update_images(&mut tracee)?; + } + Stop::SignalDelivery { + signal: Signal::SIGTRAP, + } => { + self.restore_and_call_if_breakpoint(&mut tracee)?; + } + Stop::Clone { new: pid } => { + // Only seen when the `VM_CLONE` flag is set, as of Linux 4.15. + info!("new thread: {}", pid); + } + _ => { + debug!("stop: {:?}", tracee.stop); + } + } + + if let Err(err) = self.context.tracer.restart(tracee, Restart::Syscall) { + error!("unable to restart tracee: {}", err); + } + } + + Ok(()) + } + + fn restore_and_call_if_breakpoint(&mut self, tracee: &mut Tracee) -> Result<()> { + let mut regs = tracee.registers()?; + + // Compute what the last PC would have been _if_ we stopped due to a soft breakpoint. + // + // If we don't have a registered breakpoint, then we will not use this value. + let pc = Address(regs.rip.saturating_sub(1)); + + if self.context.breakpoints.clear(tracee, pc)? { + // We restored the original, `int3`-clobbered instruction in `clear()`. Now + // set the tracee's registers to execute it on restart. Do this _before_ the + // callback to simulate a hardware breakpoint. + regs.rip = pc.0; + tracee.set_registers(regs)?; + + self.event_handler + .on_breakpoint(&mut self.context, tracee)?; + } else { + warn!("no registered breakpoint for SIGTRAP delivery at {pc:x}"); + + // We didn't fix up a registered soft breakpoint, so we have no reason to + // re-execute the instruction at the last PC. Leave the tracee registers alone. + } + + Ok(()) + } + + fn update_images(&mut self, tracee: &mut Tracee) -> Result<()> { + let images = self + .context + .images + .as_mut() + .ok_or_else(|| format_err!("internal error: recorder images not initialized"))?; + let events = images.update()?; + + for (_base, image) in &events.loaded { + self.event_handler + .on_module_load(&mut self.context, tracee, image)?; + } + + Ok(()) + } +} + +pub struct DebuggerContext { + pub breakpoints: Breakpoints, + pub images: Option, + pub tracer: Ptracer, +} + +impl DebuggerContext { + #[allow(clippy::new_without_default)] + pub fn new() -> Self { + let breakpoints = Breakpoints::default(); + let images = None; + let tracer = Ptracer::new(); + + Self { + breakpoints, + images, + tracer, + } + } + + pub fn find_image_for_addr(&self, addr: Address) -> Option<&ModuleImage> { + self.images.as_ref()?.find_image_for_addr(addr) + } +} + +/// Executable memory-mapped files for a process. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Images { + mapped: BTreeMap, + pid: i32, +} + +impl Images { + pub fn new(pid: i32) -> Self { + let mapped = BTreeMap::default(); + + Self { mapped, pid } + } + + pub fn mapped(&self) -> impl Iterator { + self.mapped.iter().map(|(va, i)| (*va, i)) + } + + pub fn update(&mut self) -> Result { + let proc = Process::new(self.pid)?; + + let mut new = BTreeMap::new(); + let mut group: Vec = vec![]; + + for map in proc.maps()? { + if let Some(last) = group.last() { + if last.pathname == map.pathname { + // The current memory mapping is the start of a new group. + // + // Consume the current group, and track any new module image. + if let Ok(image) = ModuleImage::new(group) { + let base = image.base(); + new.insert(base, image); + } + + // Reset the current group. + group = vec![]; + } + } + + group.push(map); + } + + let events = LoadEvents::new(&self.mapped, &new); + + self.mapped = new; + + Ok(events) + } + + pub fn find_image_for_addr(&self, addr: Address) -> Option<&ModuleImage> { + let (_, image) = self.mapped().find(|(_, im)| im.contains(&addr))?; + + Some(image) + } +} + +/// A `MemoryMap` that is known to be file-backed and executable. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ModuleImage { + base: Address, + maps: Vec, + path: FilePath, +} + +impl ModuleImage { + // Accepts an increasing sequence of memory mappings with a common file-backed + // pathname. + pub fn new(mut maps: Vec) -> Result { + maps.sort_by_key(|m| m.address); + + if maps.is_empty() { + bail!("no mapping for module image"); + } + + if !maps.iter().any(|m| m.perms.contains('x')) { + bail!("no executable mapping for module image"); + } + + // Cannot panic due to initial length check. + let first = &maps[0]; + + let path = if let MMapPath::Path(path) = &first.pathname { + FilePath::new(path.to_string_lossy())? + } else { + bail!("module image mappings must be file-backed"); + }; + + for map in &maps { + if map.pathname != first.pathname { + bail!("module image mapping not file-backed"); + } + } + + let base = Address(first.address.0); + + let image = ModuleImage { base, maps, path }; + + Ok(image) + } + + pub fn path(&self) -> &FilePath { + &self.path + } + + pub fn base(&self) -> Address { + self.base + } + + pub fn contains(&self, addr: &Address) -> bool { + for map in &self.maps { + let lo = Address(map.address.0); + let hi = Address(map.address.1); + if (lo..hi).contains(addr) { + return true; + } + } + + false + } +} + +pub struct LoadEvents { + pub loaded: Vec<(Address, ModuleImage)>, + pub unloaded: Vec<(Address, ModuleImage)>, +} + +impl LoadEvents { + pub fn new(old: &BTreeMap, new: &BTreeMap) -> Self { + // New not in old. + let loaded: Vec<_> = new + .iter() + .filter(|(nva, n)| { + !old.iter() + .any(|(iva, i)| *nva == iva && n.path() == i.path()) + }) + .map(|(va, i)| (*va, i.clone())) + .collect(); + + // Old not in new. + let unloaded: Vec<_> = old + .iter() + .filter(|(iva, i)| { + !new.iter() + .any(|(nva, n)| nva == *iva && n.path() == i.path()) + }) + .map(|(va, i)| (*va, i.clone())) + .collect(); + + Self { loaded, unloaded } + } +} + +#[derive(Clone, Debug, Default)] +pub struct Breakpoints { + saved: BTreeMap, +} + +impl Breakpoints { + pub fn set(&mut self, tracee: &mut Tracee, addr: Address) -> Result<()> { + // Return if the breakpoint exists. We don't want to conclude that the + // saved instruction byte was `0xcc`. + if self.saved.contains_key(&addr) { + return Ok(()); + } + + let mut data = [0u8]; + tracee.read_memory_mut(addr.0, &mut data)?; + self.saved.insert(addr, data[0]); + tracee.write_memory(addr.0, &[0xcc])?; + + Ok(()) + } + + pub fn clear(&mut self, tracee: &mut Tracee, addr: Address) -> Result { + let data = self.saved.remove(&addr); + + let cleared = if let Some(data) = data { + tracee.write_memory(addr.0, &[data])?; + true + } else { + false + }; + + Ok(cleared) + } +} + +fn continue_to_init_execve(tracer: &mut Ptracer) -> Result { + while let Some(tracee) = tracer.wait()? { + if let Stop::SyscallExit = &tracee.stop { + return Ok(tracee); + } + + tracer.restart(tracee, Restart::Continue)?; + } + + bail!("did not see initial execve() in tracee while recording coverage"); +} diff --git a/src/agent/coverage/src/record/windows.rs b/src/agent/coverage/src/record/windows.rs new file mode 100644 index 0000000000..c926889bfb --- /dev/null +++ b/src/agent/coverage/src/record/windows.rs @@ -0,0 +1,210 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::collections::BTreeMap; +use std::path::Path; +use std::process::Command; +use std::time::Duration; + +use anyhow::{anyhow, Result}; +use debuggable_module::load_module::LoadModule; +use debuggable_module::loader::Loader; +use debuggable_module::path::FilePath; +use debuggable_module::windows::WindowsModule; +use debuggable_module::Offset; +use debugger::{BreakpointId, BreakpointType, DebugEventHandler, Debugger, ModuleLoadInfo}; + +use crate::allowlist::TargetAllowList; +use crate::binary::{self, BinaryCoverage}; + +pub fn record( + cmd: Command, + timeout: Duration, + allowlist: impl Into>, +) -> Result { + let loader = Loader::new(); + let allowlist = allowlist.into().unwrap_or_default(); + + crate::timer::timed(timeout, move || { + let mut recorder = WindowsRecorder::new(&loader, allowlist); + let (mut dbg, _child) = Debugger::init(cmd, &mut recorder)?; + dbg.run(&mut recorder)?; + + Ok(recorder.coverage) + })? +} + +pub struct WindowsRecorder<'data> { + allowlist: TargetAllowList, + breakpoints: Breakpoints, + coverage: BinaryCoverage, + loader: &'data Loader, + modules: BTreeMap>, +} + +impl<'data> WindowsRecorder<'data> { + pub fn new(loader: &'data Loader, allowlist: TargetAllowList) -> Self { + let breakpoints = Breakpoints::default(); + let coverage = BinaryCoverage::default(); + let modules = BTreeMap::new(); + + Self { + allowlist, + breakpoints, + coverage, + loader, + modules, + } + } + + pub fn allowlist(&self) -> &TargetAllowList { + &self.allowlist + } + + pub fn allowlist_mut(&mut self) -> &mut TargetAllowList { + &mut self.allowlist + } + + fn try_on_create_process(&mut self, dbg: &mut Debugger, module: &ModuleLoadInfo) -> Result<()> { + // Not necessary for PDB search, but enables use of other `dbghelp` APIs. + if let Err(err) = dbg.target().maybe_sym_initialize() { + error!( + "unable to initialize symbol handler for new process {}: {:?}", + module.path().display(), + err, + ); + } + + self.insert_module(dbg, module) + } + + fn try_on_load_dll(&mut self, dbg: &mut Debugger, module: &ModuleLoadInfo) -> Result<()> { + self.insert_module(dbg, module) + } + + fn try_on_breakpoint(&mut self, _dbg: &mut Debugger, id: BreakpointId) -> Result<()> { + let breakpoint = self + .breakpoints + .remove(id) + .ok_or_else(|| anyhow!("stopped on dangling breakpoint"))?; + + let coverage = self + .coverage + .modules + .get_mut(&breakpoint.module) + .ok_or_else(|| anyhow!("coverage not initialized for module: {}", breakpoint.module))?; + + coverage.increment(breakpoint.offset)?; + + Ok(()) + } + + fn stop(&self, dbg: &mut Debugger) { + dbg.quit_debugging(); + } + + fn insert_module(&mut self, dbg: &mut Debugger, module: &ModuleLoadInfo) -> Result<()> { + let path = FilePath::new(module.path().to_string_lossy())?; + + if !self.allowlist.modules.is_allowed(&path) { + debug!("not inserting denylisted module: {path}"); + return Ok(()); + } + + let module = if let Ok(m) = WindowsModule::load(self.loader, path.clone()) { + m + } else { + debug!("skipping undebuggable module: {path}"); + return Ok(()); + }; + + let coverage = binary::find_coverage_sites(&module, &self.allowlist)?; + + for offset in coverage.as_ref().keys().copied() { + let breakpoint = Breakpoint::new(path.clone(), offset); + self.breakpoints.set(dbg, breakpoint)?; + } + + self.coverage.modules.insert(path.clone(), coverage); + + self.modules.insert(path, module); + + Ok(()) + } +} + +#[derive(Debug, Default)] +struct Breakpoints { + id_to_offset: BTreeMap, + offset_to_breakpoint: BTreeMap, +} + +impl Breakpoints { + pub fn set(&mut self, dbg: &mut Debugger, breakpoint: Breakpoint) -> Result<()> { + if self.is_set(&breakpoint) { + return Ok(()); + } + + self.write(dbg, breakpoint) + } + + // Unguarded action that ovewrites both the target process address space and our index + // of known breakpoints. Callers must use `set()`, which avoids redundant breakpoint + // setting. + fn write(&mut self, dbg: &mut Debugger, breakpoint: Breakpoint) -> Result<()> { + // The `debugger` crates tracks loaded modules by base name. If a path or file + // name is used, software breakpoints will not be written. + let name = Path::new(breakpoint.module.base_name()); + let id = dbg.new_rva_breakpoint(name, breakpoint.offset.0, BreakpointType::OneTime)?; + + self.id_to_offset.insert(id, breakpoint.offset); + self.offset_to_breakpoint + .insert(breakpoint.offset, breakpoint); + + Ok(()) + } + + pub fn is_set(&self, breakpoint: &Breakpoint) -> bool { + self.offset_to_breakpoint.contains_key(&breakpoint.offset) + } + + pub fn remove(&mut self, id: BreakpointId) -> Option { + let offset = self.id_to_offset.remove(&id)?; + self.offset_to_breakpoint.remove(&offset) + } +} + +#[derive(Clone, Debug)] +struct Breakpoint { + module: FilePath, + offset: Offset, +} + +impl Breakpoint { + pub fn new(module: FilePath, offset: Offset) -> Self { + Self { module, offset } + } +} + +impl<'data> DebugEventHandler for WindowsRecorder<'data> { + fn on_create_process(&mut self, dbg: &mut Debugger, module: &ModuleLoadInfo) { + if let Err(err) = self.try_on_create_process(dbg, module) { + warn!("{err}"); + self.stop(dbg); + } + } + + fn on_load_dll(&mut self, dbg: &mut Debugger, module: &ModuleLoadInfo) { + if let Err(err) = self.try_on_load_dll(dbg, module) { + warn!("{err}"); + self.stop(dbg); + } + } + + fn on_breakpoint(&mut self, dbg: &mut Debugger, bp: BreakpointId) { + if let Err(err) = self.try_on_breakpoint(dbg, bp) { + warn!("{err}"); + self.stop(dbg); + } + } +} diff --git a/src/agent/coverage/src/source.rs b/src/agent/coverage/src/source.rs new file mode 100644 index 0000000000..33067e6cc6 --- /dev/null +++ b/src/agent/coverage/src/source.rs @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::collections::{BTreeMap, BTreeSet}; + +use anyhow::{bail, Result}; + +use debuggable_module::block::{sweep_region, Block, Blocks}; +use debuggable_module::load_module::LoadModule; +use debuggable_module::loader::Loader; +use debuggable_module::path::FilePath; +use debuggable_module::{Module, Offset}; + +use crate::binary::BinaryCoverage; + +pub use crate::binary::Count; + +#[derive(Clone, Debug, Default)] +pub struct SourceCoverage { + pub files: BTreeMap, +} + +#[derive(Clone, Debug, Default)] +pub struct FileCoverage { + pub lines: BTreeMap, +} + +// Must be nonzero. +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct Line(u32); + +impl Line { + pub fn new(number: u32) -> Result { + if number == 0 { + bail!("line numbers must be nonzero"); + } + + Ok(Line(number)) + } + + pub fn number(&self) -> u32 { + self.0 + } +} + +impl From for u32 { + fn from(line: Line) -> Self { + line.number() + } +} + +pub fn binary_to_source_coverage(binary: &BinaryCoverage) -> Result { + use std::collections::btree_map::Entry; + + use symbolic::debuginfo::Object; + use symbolic::symcache::{SymCache, SymCacheConverter}; + + let loader = Loader::new(); + + let mut source = SourceCoverage::default(); + + for (exe_path, coverage) in &binary.modules { + let module: Box = Box::load(&loader, exe_path.clone())?; + let debuginfo = module.debuginfo()?; + + let mut symcache = vec![]; + let mut converter = SymCacheConverter::new(); + + let exe = Object::parse(module.executable_data())?; + converter.process_object(&exe)?; + + let di = Object::parse(module.debuginfo_data())?; + converter.process_object(&di)?; + + converter.serialize(&mut std::io::Cursor::new(&mut symcache))?; + let symcache = SymCache::parse(&symcache)?; + + let mut blocks = Blocks::new(); + + for function in debuginfo.functions() { + for offset in coverage.as_ref().keys() { + // Recover function blocks if it contains any coverage offset. + if function.contains(offset) { + let function_blocks = + sweep_region(&*module, &debuginfo, function.offset, function.size)?; + blocks.extend(&function_blocks); + break; + } + } + } + + for (offset, count) in coverage.as_ref() { + // Inflate blocks. + if let Some(block) = blocks.find(offset) { + let block_offsets = instruction_offsets(&*module, block)?; + + for offset in block_offsets { + for location in symcache.lookup(offset.0) { + let line_number = location.line(); + + if line_number == 0 { + continue; + } + + if let Some(file) = location.file() { + let file_path = FilePath::new(file.full_path())?; + + // We have a hit. + let file_coverage = source.files.entry(file_path).or_default(); + let line = Line(line_number); + + match file_coverage.lines.entry(line) { + Entry::Occupied(occupied) => { + let old = occupied.into_mut(); + + // If we miss any part of a line, count it as missed. + let new = u32::max(old.0, count.0); + + *old = Count(new); + } + Entry::Vacant(vacant) => { + vacant.insert(*count); + } + } + } + } + } + } + } + } + + Ok(source) +} + +fn instruction_offsets(module: &dyn Module, block: &Block) -> Result> { + use iced_x86::Decoder; + let data = module.read(block.offset, block.size)?; + + let mut offsets: BTreeSet = BTreeSet::default(); + + let mut pc = block.offset.0; + let mut decoder = Decoder::new(64, data, 0); + decoder.set_ip(pc); + + while decoder.can_decode() { + let inst = decoder.decode(); + + if inst.is_invalid() { + break; + } + + offsets.insert(Offset(pc)); + pc = inst.ip(); + } + + Ok(offsets) +} diff --git a/src/agent/coverage/src/timer.rs b/src/agent/coverage/src/timer.rs new file mode 100644 index 0000000000..24c8b44c51 --- /dev/null +++ b/src/agent/coverage/src/timer.rs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::sync::mpsc; +use std::thread; +use std::time::Duration; + +use anyhow::{bail, Result}; + +pub fn timed(timeout: Duration, function: F) -> Result +where + T: Send + 'static, + F: FnOnce() -> T + Send + 'static, +{ + let (worker_sender, receiver) = mpsc::channel(); + let timer_sender = worker_sender.clone(); + + let _worker = thread::spawn(move || { + let out = function(); + worker_sender.send(Timed::Done(out)).unwrap(); + }); + + let _timer = thread::spawn(move || { + thread::sleep(timeout); + timer_sender.send(Timed::Timeout).unwrap(); + }); + + match receiver.recv()? { + Timed::Done(out) => Ok(out), + Timed::Timeout => bail!("function exceeded timeout of {:?}", timeout), + } +} + +enum Timed { + Done(T), + Timeout, +} diff --git a/src/agent/debuggable-module/Cargo.toml b/src/agent/debuggable-module/Cargo.toml new file mode 100644 index 0000000000..27f72c8a31 --- /dev/null +++ b/src/agent/debuggable-module/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "debuggable-module" +version = "0.1.0" +edition = "2021" +license = "MIT" + +[dependencies] +anyhow = "1.0" +gimli = "0.26.2" +goblin = "0.6.0" +iced-x86 = "1.17" +log = "0.4.17" +pdb = "0.8.0" +regex = "1.0" +symbolic = { version = "10.1", features = ["debuginfo", "demangle", "symcache"] } +thiserror = "1.0" + +[dev-dependencies] +clap = { version = "4.0", features = ["derive"] } diff --git a/src/agent/debuggable-module/examples/dump.rs b/src/agent/debuggable-module/examples/dump.rs new file mode 100644 index 0000000000..2d5e742c9d --- /dev/null +++ b/src/agent/debuggable-module/examples/dump.rs @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#![allow(clippy::all)] + +use std::rc::Rc; + +use anyhow::Result; +use clap::Parser; + +use debuggable_module::{ + block, + debuginfo::{DebugInfo, Function}, + load_module::LoadModule, + loader::Loader, + path::FilePath, + {Module, Offset}, +}; +use iced_x86::{Decoder, Formatter, Instruction, NasmFormatter, SymbolResolver, SymbolResult}; +use regex::Regex; + +#[derive(Parser, Debug)] +struct Args { + #[arg(short, long)] + module: String, + + #[arg(short, long)] + function: Option, +} + +fn main() -> Result<()> { + let args = Args::parse(); + + let loader = Loader::new(); + let path = FilePath::new(&args.module)?; + let module = Box::::load(&loader, path)?; + let debuginfo = Rc::new(module.debuginfo()?); + + let glob = args.function.unwrap_or(".*".to_owned()); + let regex = glob_to_regex(&glob)?; + + for function in debuginfo.functions() { + if regex.is_match(&function.name) { + dump_function(&*module, debuginfo.clone(), function)?; + } + } + + Ok(()) +} + +fn glob_to_regex(expr: &str) -> Result { + // Don't make users escape Windows path separators. + let expr = expr.replace(r"\", r"\\"); + + // Translate glob wildcards into quantified regexes. + let expr = expr.replace("*", ".*"); + + // Anchor to line start. + let expr = format!("^{expr}"); + + Ok(Regex::new(&expr)?) +} + +fn dump_function(module: &dyn Module, debuginfo: Rc, function: &Function) -> Result<()> { + println!("{}", function.name); + + let mut fmt = formatter(debuginfo.clone()); + + let blocks = block::sweep_region(module, &*debuginfo, function.offset, function.size)?; + + for block in &blocks { + let data = module.read(block.offset, block.size)?; + dump(block.offset.0, data, &mut *fmt)?; + println!() + } + + Ok(()) +} + +fn dump(pc: u64, data: &[u8], fmt: &mut dyn Formatter) -> Result<()> { + let mut decoder = Decoder::new(64, data, 0); + decoder.set_ip(pc); + + while decoder.can_decode() { + let inst = decoder.decode(); + + let mut display = String::new(); + fmt.format(&inst, &mut display); + println!("{:>12x} {}", inst.ip(), display); + } + + Ok(()) +} + +fn formatter(debuginfo: Rc) -> Box { + let resolver = Box::new(Resolver(debuginfo)); + let mut fmt = NasmFormatter::with_options(Some(resolver), None); + + let opts = fmt.options_mut(); + opts.set_add_leading_zero_to_hex_numbers(false); + opts.set_branch_leading_zeros(false); + opts.set_displacement_leading_zeros(false); + opts.set_hex_prefix("0x"); + opts.set_hex_suffix(""); + opts.set_leading_zeros(false); + opts.set_uppercase_all(false); + opts.set_uppercase_hex(false); + opts.set_space_after_operand_separator(true); + opts.set_space_between_memory_add_operators(true); + + Box::new(fmt) +} + +struct Resolver(Rc); + +impl SymbolResolver for Resolver { + fn symbol( + &mut self, + _instruction: &Instruction, + _operand: u32, + _instruction_operand: Option, + address: u64, + _address_size: u32, + ) -> Option { + let f = self.0.find_function(Offset(address))?; + + Some(SymbolResult::with_str(f.offset.0, &f.name)) + } +} diff --git a/src/agent/debuggable-module/src/block.rs b/src/agent/debuggable-module/src/block.rs new file mode 100644 index 0000000000..b4fc098ea3 --- /dev/null +++ b/src/agent/debuggable-module/src/block.rs @@ -0,0 +1,286 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use anyhow::Result; +use iced_x86::Decoder; +use std::collections::{BTreeMap, BTreeSet}; + +use crate::debuginfo::DebugInfo; +use crate::{Module, Offset}; + +pub fn sweep_module<'data>(module: &dyn Module<'data>, debuginfo: &DebugInfo) -> Result { + let mut blocks = Blocks::default(); + + for function in debuginfo.functions() { + let function_blocks = sweep_region(module, debuginfo, function.offset, function.size)?; + blocks.map.extend(&function_blocks.map); + } + + Ok(blocks) +} + +pub fn sweep_region<'data>( + module: &dyn Module<'data>, + debuginfo: &DebugInfo, + offset: Offset, + size: u64, +) -> Result { + use iced_x86::Code; + use iced_x86::FlowControl::*; + + let region = offset.region(size); + + let data = module.read(offset, size)?; + let mut decoder = Decoder::new(64, data, 0); + + let mut visited = BTreeSet::new(); + + let mut pending = Vec::new(); + + // Schedule the function entrypoint. + pending.push(offset.0); + + // Schedule any extra jump labels in the target region. + for label in debuginfo.labels() { + // Don't duplicate function entrypoint. + if label == offset { + continue; + } + + // Don't visit labels outside of the function region. + if !region.contains(&label.0) { + continue; + } + + pending.push(label.0); + } + + while let Some(entry) = pending.pop() { + if !region.contains(&entry) { + continue; + } + + if visited.contains(&entry) { + continue; + } + + visited.insert(entry); + + // Reset decoder for `entry`. + let position = (entry - offset.0) as usize; + decoder.set_position(position)?; + decoder.set_ip(entry); + + // Decode instructions (starting from `entry`) until we reach a block + // terminator or run out of valid data. + while decoder.can_decode() { + let inst = decoder.decode(); + + match inst.flow_control() { + IndirectBranch => { + // Treat as an unconditional branch, discarding indirect target. + break; + } + UnconditionalBranch => { + // Target is an entrypoint. + let target = inst.ip_rel_memory_address(); + pending.push(target); + + // We can't fall through to the next instruction, so don't add it to + // the worklist. + break; + } + ConditionalBranch => { + // Target is an entrypoint. + let target = inst.ip_rel_memory_address(); + pending.push(target); + + // We can fall through, so add to work list. + pending.push(inst.next_ip()); + + // Fall through not guaranteed, so this block is terminated. + break; + } + Return => { + break; + } + Call => { + let target = Offset(inst.near_branch_target()); + + // If call site is `noreturn`, then next instruction is not reachable. + let noreturn = debuginfo + .functions() + .find(|f| f.contains(&target)) + .map(|f| f.noreturn) + .unwrap_or(false); + + if noreturn { + break; + } + } + Exception => { + // Invalid instruction or UD. + break; + } + Interrupt => { + if inst.code() == Code::Int3 { + // Treat as noreturn function call. + break; + } + } + Next => { + // Fall through. + } + IndirectCall => { + // We dont' know the callee and can't tell if it is `noreturn`, so fall through. + } + XbeginXabortXend => { + // Not yet analyzed, so fall through. + } + } + } + } + + let mut blocks = Blocks::default(); + + for &entry in &visited { + // Reset decoder for `entry`. + let position = (entry - offset.0) as usize; + decoder.set_position(position)?; + decoder.set_ip(entry); + + while decoder.can_decode() { + let inst = decoder.decode(); + + if inst.is_invalid() { + // Assume that the decoder PC is in an undefined state. Reset it so we can + // just query the decoder to get the exclusive upper bound on loop exit. + decoder.set_ip(inst.ip()); + break; + } + + match inst.flow_control() { + IndirectBranch => { + break; + } + UnconditionalBranch => { + break; + } + ConditionalBranch => { + break; + } + Return => { + break; + } + Call => { + let target = Offset(inst.near_branch_target()); + + // If call site is `noreturn`, then next instruction is not reachable. + let noreturn = debuginfo + .functions() + .find(|f| f.contains(&target)) + .map(|f| f.noreturn) + .unwrap_or(false); + + if noreturn { + break; + } + } + Exception => { + // Ensure that the decoder PC points to the first instruction outside + // of the block. + // + // By doing this, we always exclude UD instructions from blocks. + decoder.set_ip(inst.ip()); + + // Invalid instruction or UD. + break; + } + Interrupt => { + if inst.code() == Code::Int3 { + // Treat as noreturn function call. + break; + } + } + Next => { + // Fall through. + } + IndirectCall => { + // We dont' know the callee and can't tell if it is `noreturn`, so fall through. + } + XbeginXabortXend => { + // Not yet analyzed, so fall through. + } + } + + // Based only on instruction semantics, we'd continue. But if the + // next offset is a known block entrypoint, we're at a terminator. + if visited.contains(&inst.next_ip()) { + break; + } + } + + let end = decoder.ip(); + let size = end.saturating_sub(entry); + + if size > 0 { + let offset = Offset(entry); + let block = Block::new(offset, size); + blocks.map.insert(offset, block); + } else { + warn!("dropping empty block {:x}..{:x}", entry, end); + } + } + + Ok(blocks) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct Block { + pub offset: Offset, + pub size: u64, +} + +impl Block { + pub fn new(offset: Offset, size: u64) -> Self { + Self { offset, size } + } + + pub fn contains(&self, offset: &Offset) -> bool { + self.offset.region(self.size).contains(&offset.0) + } +} + +#[derive(Clone, Debug, Default)] +pub struct Blocks { + pub map: BTreeMap, +} + +impl Blocks { + pub fn new() -> Self { + Self::default() + } + + pub fn iter(&self) -> impl Iterator { + self.map.values() + } + + pub fn find(&self, offset: &Offset) -> Option<&Block> { + self.map.values().find(|b| b.contains(offset)) + } + + pub fn extend<'b>(&mut self, blocks: impl IntoIterator) { + for &b in blocks.into_iter() { + self.map.insert(b.offset, b); + } + } +} + +impl<'b> IntoIterator for &'b Blocks { + type Item = &'b Block; + type IntoIter = std::collections::btree_map::Values<'b, Offset, Block>; + + fn into_iter(self) -> Self::IntoIter { + self.map.values() + } +} diff --git a/src/agent/debuggable-module/src/debuginfo.rs b/src/agent/debuggable-module/src/debuginfo.rs new file mode 100644 index 0000000000..7411d8bb06 --- /dev/null +++ b/src/agent/debuggable-module/src/debuginfo.rs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::collections::{BTreeMap, BTreeSet}; +use std::ops::Range; + +use crate::Offset; + +pub struct DebugInfo { + functions: BTreeMap, + labels: BTreeSet, +} + +impl DebugInfo { + pub fn new(functions: BTreeMap, labels: Option>) -> Self { + let labels = labels.unwrap_or_default(); + + Self { functions, labels } + } + + pub fn functions(&self) -> impl Iterator { + self.functions.values() + } + + pub fn labels(&self) -> impl Iterator + '_ { + self.labels.iter().copied() + } + + #[allow(clippy::manual_find)] + pub fn find_function(&self, offset: Offset) -> Option<&Function> { + // Search backwards from first function whose entrypoint is less than or + // equal to `offset`. + for f in self.functions.range(..=offset).map(|(_, f)| f).rev() { + if f.contains(&offset) { + return Some(f); + } + } + + None + } +} + +pub struct Function { + pub name: String, + pub offset: Offset, + pub size: u64, + pub noreturn: bool, +} + +impl Function { + pub fn contains(&self, offset: &Offset) -> bool { + let range = self.offset.region(self.size); + range.contains(&offset.0) + } + + pub fn range(&self) -> Range { + let lo = self.offset; + let hi = Offset(lo.0.saturating_add(self.size)); + lo..hi + } +} diff --git a/src/agent/debuggable-module/src/lib.rs b/src/agent/debuggable-module/src/lib.rs new file mode 100644 index 0000000000..1e71706d11 --- /dev/null +++ b/src/agent/debuggable-module/src/lib.rs @@ -0,0 +1,124 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#[macro_use] +extern crate log; + +use std::fmt; +use std::ops::Range; + +use anyhow::{anyhow as error, Result}; + +pub mod block; +pub mod debuginfo; +pub mod linux; +pub mod load_module; +pub mod loader; +pub mod path; +pub mod windows; + +use crate::debuginfo::DebugInfo; +use crate::path::FilePath; + +/// Executable code module, with debuginfo. +/// +/// The debuginfo may be inline, or split into a separate file. +pub trait Module<'data> { + /// Path to the executable module file. + fn executable_path(&self) -> &FilePath; + + /// Path to the file containing debug info for the executable. + /// + /// May be the same as the executable path. + fn debuginfo_path(&self) -> &FilePath; + + /// Read `size` bytes of data from the module-relative virtual offset `offset`. + /// + /// Will return an error if the requested region is outside of the range of the + /// module's image. + fn read(&self, offset: Offset, size: u64) -> Result<&'data [u8]>; + + /// Nominal base load address of the module image. + fn base_address(&self) -> Address; + + /// Raw bytes of the executable file. + fn executable_data(&self) -> &'data [u8]; + + /// Raw bytes of the file that contains debug info. + /// + /// May be the same as the executable data. + fn debuginfo_data(&self) -> &'data [u8]; + + /// Debugging information derived from the module and its debuginfo. + fn debuginfo(&self) -> Result; +} + +/// Virtual address. +/// +/// May be used to represent an internal fiction of debuginfo, or real address image. In +/// any case, the virtual address is absolute, not module-relative. +/// +/// No validity assumption can be made about the address value. For example, it may be: +/// - Zero +/// - In either userspace or kernelspace +/// - Non-canonical for x86-64 +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct Address(pub u64); + +impl Address { + /// Returns the address located `offset` bytes above `self`. + /// + /// Can be used to convert a module-relative offset to a virtual address by adding it + /// to a module's base virtual address. + /// + /// Fails if the new address is not representable by a `u64`. + pub fn offset_by(&self, offset: Offset) -> Result
{ + let addr = self + .0 + .checked_add(offset.0) + .ok_or_else(|| error!("overflow: {:x} + {:x}", self.0, offset.0))?; + + Ok(Address(addr)) + } + + /// Returns `self` as an `addr`-relative `offset`. + /// + /// Can be used to convert a virtual address to a module-relative offset by + /// subtracting the module's base virtual address. + /// + /// Fails if `self` < `addr`. + pub fn offset_from(&self, addr: Address) -> Result { + let offset = self + .0 + .checked_sub(addr.0) + .ok_or_else(|| error!("underflow: {:x} - {:x}", self.0, addr.0))?; + + Ok(Offset(offset)) + } +} + +impl fmt::LowerHex for Address { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{:x}", self.0) + } +} + +/// Positive byte offset from some byte location. +/// +/// May be relative to a virtual address, another virtual offset, or a file position. +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct Offset(pub u64); + +impl Offset { + pub fn region(&self, size: u64) -> Range { + let lo = self.0; + let hi = lo.saturating_add(size); + lo..hi + } +} + +impl fmt::LowerHex for Offset { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{:x}", self.0) + } +} diff --git a/src/agent/debuggable-module/src/linux.rs b/src/agent/debuggable-module/src/linux.rs new file mode 100644 index 0000000000..951505e686 --- /dev/null +++ b/src/agent/debuggable-module/src/linux.rs @@ -0,0 +1,268 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::collections::{BTreeMap, BTreeSet}; +use std::ops::Range; + +use anyhow::{bail, Result}; +use gimli::{EndianSlice, LittleEndian, SectionId}; +use goblin::elf::{program_header::PT_LOAD, Elf, ProgramHeader}; + +use crate::debuginfo::{DebugInfo, Function}; +use crate::path::FilePath; +use crate::{Address, Module, Offset}; + +impl<'data> Module<'data> for LinuxModule<'data> { + fn executable_path(&self) -> &FilePath { + &self.path + } + + fn debuginfo_path(&self) -> &FilePath { + &self.path + } + + fn read(&self, offset: Offset, size: u64) -> Result<&'data [u8]> { + if size == 0 { + return Ok(&[]); + } + + let addr = Address(self.vmmap.base()).offset_by(offset)?.0; + + for segment in self.vmmap.segments.values() { + if segment.vm_range.contains(&addr) { + // Segment-relative offset of the virtual address. + let seg_off = addr.saturating_sub(segment.vm_range.start); + + let file_offset = segment.file_range.start + seg_off; + let available = segment.file_size().saturating_sub(seg_off); + let read_size = u64::min(available, size); + + let lo = file_offset as usize; + let hi = file_offset.saturating_add(read_size) as usize; + + return Ok(&self.data[lo..hi]); + } + } + + bail!("no data for VM offset: {:x}", offset.0); + } + + fn base_address(&self) -> Address { + Address(self.vmmap.base()) + } + + fn executable_data(&self) -> &'data [u8] { + self.data + } + + fn debuginfo_data(&self) -> &'data [u8] { + // Assume embedded DWARF. + self.data + } + + fn debuginfo(&self) -> Result { + use symbolic::debuginfo::Object; + use symbolic::demangle::{Demangle, DemangleOptions}; + + let noreturns = self.noreturns()?; + let opts = DemangleOptions::complete(); + + let object = Object::parse(self.debuginfo_data())?; + let session = object.debug_session()?; + + let mut functions = BTreeMap::new(); + + for function in session.functions() { + let function = function?; + + let name = function.name.try_demangle(opts).into_owned(); + let offset = Offset(function.address); // Misnamed. + let size = function.size; + let noreturn = noreturns.contains(&offset); + + let f = Function { + name, + noreturn, + offset, + size, + }; + functions.insert(offset, f); + } + + Ok(DebugInfo::new(functions, None)) + } +} + +pub struct LinuxModule<'data> { + path: FilePath, + data: &'data [u8], + elf: Elf<'data>, + vmmap: VmMap, +} + +impl<'data> LinuxModule<'data> { + pub fn new(path: FilePath, data: &'data [u8]) -> Result { + let elf = Elf::parse(data)?; + let vmmap = VmMap::new(&elf)?; + + Ok(Self { + path, + data, + elf, + vmmap, + }) + } + + pub fn elf(&self) -> &Elf<'data> { + &self.elf + } + + fn noreturns(&self) -> Result> { + use gimli::{AttributeValue, DW_AT_low_pc, DW_AT_noreturn, DW_TAG_subprogram, Dwarf}; + + let loader = |s| self.load_section(s); + let dwarf = Dwarf::load(loader)?; + + let mut noreturns = BTreeSet::new(); + + // Iterate over all compilation units. + let mut headers = dwarf.units(); + while let Some(header) = headers.next()? { + let unit = dwarf.unit(header)?; + + let mut entries = unit.entries(); + while let Some((_, entry)) = entries.next_dfs()? { + // Look for `noreturn` functions. + if entry.tag() == DW_TAG_subprogram { + let mut low_pc = None; + + // Find the virtual function low_pc offset. + let mut attrs = entry.attrs(); + while let Some(attr) = attrs.next()? { + if attr.name() == DW_AT_low_pc { + let value = attr.value(); + + if let AttributeValue::Addr(value) = value { + // `low_pc` is 0 for inlines. + if value != 0 { + low_pc = Some(value); + break; + } + } + } + } + + let low_pc = if let Some(low_pc) = low_pc { + low_pc + } else { + // No low PC, can't locate subprogram. + continue; + }; + + let mut attrs = entry.attrs(); + while let Some(attr) = attrs.next()? { + if attr.name() == DW_AT_noreturn { + let value = attr.value(); + + if let AttributeValue::Flag(is_set) = value { + if is_set { + let base = Address(self.vmmap.base()); + let offset = Address(low_pc).offset_from(base)?; + noreturns.insert(offset); + break; + } + } + } + } + } + } + } + + Ok(noreturns) + } + + fn load_section(&self, section: SectionId) -> Result> { + for shdr in &self.elf.section_headers { + if let Some(name) = self.elf.shdr_strtab.get_at(shdr.sh_name) { + if name == section.name() { + if let Some(range) = shdr.file_range() { + if let Some(data) = self.data.get(range) { + let data = EndianSlice::new(data, LittleEndian); + return Ok(data); + } + } + } + } + } + + let data = EndianSlice::new(&[], LittleEndian); + Ok(data) + } +} + +struct VmMap { + base: u64, + segments: BTreeMap, +} + +impl VmMap { + pub fn new(elf: &Elf) -> Result { + let mut segments = BTreeMap::new(); + + for header in &elf.program_headers { + if header.p_type == PT_LOAD { + let segment = Segment::from(header); + segments.insert(segment.base(), segment); + } + } + + let base = *segments.keys().next().unwrap(); + + Ok(Self { base, segments }) + } + + pub fn base(&self) -> u64 { + self.base + } +} + +#[derive(Clone, Debug)] +pub struct Segment { + vm_range: Range, // VM range may exceed file range. + file_range: Range, +} + +impl Segment { + pub fn base(&self) -> u64 { + self.vm_range.start + } + + pub fn vm_size(&self) -> u64 { + self.vm_range.end - self.vm_range.start + } + + pub fn file_size(&self) -> u64 { + self.file_range.end - self.file_range.start + } +} + +impl<'h> From<&'h ProgramHeader> for Segment { + fn from(header: &'h ProgramHeader) -> Self { + let file_range = { + let lo = header.p_offset; + let hi = lo.saturating_add(header.p_filesz); + lo..hi + }; + + let vm_range = { + let lo = header.p_vaddr; + let hi = lo.saturating_add(header.p_memsz); + lo..hi + }; + + Self { + file_range, + vm_range, + } + } +} diff --git a/src/agent/debuggable-module/src/load_module.rs b/src/agent/debuggable-module/src/load_module.rs new file mode 100644 index 0000000000..982671b8b0 --- /dev/null +++ b/src/agent/debuggable-module/src/load_module.rs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::io::Cursor; + +use anyhow::{bail, Result}; +use goblin::Hint; + +use crate::linux::LinuxModule; +use crate::loader::Loader; +use crate::path::FilePath; +use crate::windows::WindowsModule; +use crate::Module; + +pub trait LoadModule<'data> +where + Self: Sized, +{ + fn load(loader: &'data Loader, exe_path: FilePath) -> Result; +} + +impl<'data> LoadModule<'data> for LinuxModule<'data> { + fn load(loader: &'data Loader, elf_path: FilePath) -> Result { + let data = loader.load(&elf_path)?; + LinuxModule::new(elf_path, data) + } +} + +impl<'data> LoadModule<'data> for WindowsModule<'data> { + fn load(loader: &'data Loader, pe_path: FilePath) -> Result { + let pdb_path = find_pdb(&pe_path)?; + let pdb_data = loader.load(&pdb_path)?; + let pe_data = loader.load(&pe_path)?; + + WindowsModule::new(pe_path, pe_data, pdb_path, pdb_data) + } +} + +impl<'data> LoadModule<'data> for Box + 'data> { + fn load(loader: &'data Loader, exe_path: FilePath) -> Result { + let exe_data = loader.load(&exe_path)?; + + let mut cursor = Cursor::new(&exe_data); + let hint = goblin::peek(&mut cursor)?; + + let module: Box> = match hint { + Hint::Elf(..) => { + let module = LinuxModule::load(loader, exe_path)?; + Box::new(module) + } + Hint::PE => { + let module = WindowsModule::load(loader, exe_path)?; + Box::new(module) + } + _ => { + bail!("unknown module file format: {:x?}", hint); + } + }; + + Ok(module) + } +} + +fn find_pdb(pe_path: &FilePath) -> Result { + // Check if the PDB is in the same dir as the PE. + let same_dir_path = pe_path.with_extension("pdb"); + + if same_dir_path.as_path().exists() { + return FilePath::new(same_dir_path); + } + + bail!("could not find PDB for PE `{pe_path}`"); +} diff --git a/src/agent/debuggable-module/src/loader.rs b/src/agent/debuggable-module/src/loader.rs new file mode 100644 index 0000000000..2998e95a2a --- /dev/null +++ b/src/agent/debuggable-module/src/loader.rs @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::collections::HashMap; +use std::sync::Mutex; + +use anyhow::Result; +use thiserror::Error; + +use crate::path::FilePath; + +#[derive(Clone, Copy)] +struct Leaked(&'static [u8]); + +impl Leaked { + pub fn into_raw(self) -> *mut [u8] { + self.0 as *const _ as *mut _ + } +} + +impl From> for Leaked { + fn from(data: Vec) -> Self { + let data = Box::leak(data.into_boxed_slice()); + Leaked(data) + } +} + +#[derive(Default)] +pub struct Loader { + loaded: Mutex>, +} + +impl Loader { + pub fn new() -> Self { + Self::default() + } + + pub fn load(&self, path: &FilePath) -> Result<&[u8]> { + if let Some(data) = self.get(path)? { + Ok(data) + } else { + self.load_new(path) + } + } + + fn load_new(&self, path: &FilePath) -> Result<&[u8]> { + let mut loaded = self.loaded.lock().map_err(|_| LoaderError::PoisonedMutex)?; + let data = std::fs::read(path)?; + let leaked = Leaked::from(data); + loaded.insert(path.clone(), leaked); + + Ok(leaked.0) + } + + pub fn get(&self, path: &FilePath) -> Result> { + let loaded = self.loaded.lock().map_err(|_| LoaderError::PoisonedMutex)?; + + let data = loaded.get(path).map(|l| l.0); + + Ok(data) + } +} + +impl Drop for Loader { + fn drop(&mut self) { + if let Ok(mut loaded) = self.loaded.lock() { + for (_, leaked) in loaded.drain() { + unsafe { + let raw = leaked.into_raw(); + let owned = Box::from_raw(raw); + drop(owned); + } + } + + debug_assert!(loaded.is_empty()); + } + } +} + +#[derive(Error, Debug)] +pub enum LoaderError { + #[error("internal mutex poisoned")] + PoisonedMutex, +} diff --git a/src/agent/debuggable-module/src/path.rs b/src/agent/debuggable-module/src/path.rs new file mode 100644 index 0000000000..7f9e8b98ac --- /dev/null +++ b/src/agent/debuggable-module/src/path.rs @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::ffi::OsStr; +use std::fmt; +use std::path::{Path, PathBuf}; + +use anyhow::{bail, Result}; + +/// Path to a file. Guaranteed UTF-8. +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct FilePath(String); + +impl FilePath { + pub fn new(path: impl Into) -> Result { + let path = path.into(); + + if Path::new(&path).file_name().is_none() { + bail!("module path has no file name"); + } + + if Path::new(&path).file_stem().is_none() { + bail!("module path has no file stem"); + } + + Ok(Self(path)) + } + + pub fn with_extension(&self, extension: impl AsRef) -> Self { + let path = self + .as_path() + .with_extension(extension.as_ref()) + .to_string_lossy() + .into_owned(); + + Self(path) + } + + pub fn as_path(&self) -> &Path { + Path::new(&self.0) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn file_name(&self) -> &str { + // Unwraps checked by ctor. + Path::new(&self.0).file_name().unwrap().to_str().unwrap() + } + + pub fn directory(&self) -> &str { + // Unwraps checked by ctor. + Path::new(&self.0).parent().unwrap().to_str().unwrap() + } + + pub fn base_name(&self) -> &str { + // Unwraps checked by ctor. + Path::new(&self.0).file_stem().unwrap().to_str().unwrap() + } +} + +impl From for String { + fn from(path: FilePath) -> Self { + path.0 + } +} +impl From for PathBuf { + fn from(path: FilePath) -> Self { + path.0.into() + } +} + +impl AsRef for FilePath { + fn as_ref(&self) -> &str { + self.as_str() + } +} + +impl AsRef for FilePath { + fn as_ref(&self) -> &OsStr { + self.as_str().as_ref() + } +} + +impl AsRef for FilePath { + fn as_ref(&self) -> &Path { + self.as_path() + } +} + +impl fmt::Display for FilePath { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.0) + } +} diff --git a/src/agent/debuggable-module/src/windows.rs b/src/agent/debuggable-module/src/windows.rs new file mode 100644 index 0000000000..a50b7bc1b9 --- /dev/null +++ b/src/agent/debuggable-module/src/windows.rs @@ -0,0 +1,232 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::cell::{Ref, RefCell}; +use std::collections::{BTreeMap, BTreeSet}; +use std::io::Cursor; + +use anyhow::Result; +use goblin::pe::PE; +use pdb::{AddressMap, ImageSectionHeader, PdbInternalSectionOffset, PDB}; + +use crate::debuginfo::{DebugInfo, Function}; +use crate::path::FilePath; +use crate::{Address, Module, Offset}; + +impl<'data> Module<'data> for WindowsModule<'data> { + fn executable_path(&self) -> &FilePath { + &self.pe_path + } + + fn debuginfo_path(&self) -> &FilePath { + &self.pdb_path + } + + fn read(&self, offset: Offset, size: u64) -> Result<&'data [u8]> { + let size = usize::try_from(size)?; + + let start = self.translator.virtual_offset_to_file_offset(offset)?; + let lo = usize::try_from(start)?; + + let end = lo.saturating_add(size); + let ub = self.pe_data.len(); + let hi = usize::min(ub, end); + + Ok(&self.pe_data[lo..hi]) + } + + fn base_address(&self) -> Address { + self.translator.base + } + + fn executable_data(&self) -> &'data [u8] { + self.pe_data + } + + fn debuginfo_data(&self) -> &'data [u8] { + self.pdb_data + } + + fn debuginfo(&self) -> Result { + use symbolic::debuginfo::Object; + use symbolic::demangle::{Demangle, DemangleOptions}; + + let extra = self.extra_debug_info()?; + let opts = DemangleOptions::complete(); + + let object = Object::parse(self.debuginfo_data())?; + let session = object.debug_session()?; + + let mut functions = BTreeMap::new(); + + for function in session.functions() { + let function = function?; + + let name = function.name.try_demangle(opts).into_owned(); + let offset = Offset(function.address); // Misnamed. + let size = function.size; + let noreturn = extra.noreturns.contains(&offset); + + let f = Function { + name, + noreturn, + offset, + size, + }; + functions.insert(offset, f); + } + + Ok(DebugInfo::new(functions, Some(extra.labels))) + } +} + +pub struct WindowsModule<'data> { + pe: PE<'data>, + pe_data: &'data [u8], + pe_path: FilePath, + + pdb: RefCell>>, + pdb_data: &'data [u8], + pdb_path: FilePath, + + translator: Translator<'data>, +} + +impl<'data> WindowsModule<'data> { + pub fn new( + pe_path: FilePath, + pe_data: &'data [u8], + pdb_path: FilePath, + pdb_data: &'data [u8], + ) -> Result { + let pe = goblin::pe::PE::parse(pe_data)?; + let mut pdb = pdb::PDB::open(Cursor::new(pdb_data))?; + + let base = Address(u64::try_from(pe.image_base)?); + let translator = Translator::new(base, &mut pdb)?; + + let pdb = RefCell::new(pdb); + + Ok(Self { + pe, + pe_data, + pe_path, + pdb, + pdb_data, + pdb_path, + translator, + }) + } + + pub fn pe(&self) -> &PE<'data> { + &self.pe + } + + pub fn pdb(&self) -> Ref>> { + self.pdb.borrow() + } + + fn extra_debug_info(&self) -> Result { + use pdb::{FallibleIterator, SymbolData}; + + let mut extra = ExtraDebugInfo::default(); + + let mut pdb = self.pdb.borrow_mut(); + + let di = pdb.debug_information()?; + let mut modules = di.modules()?; + + while let Some(module) = modules.next()? { + if let Some(mi) = pdb.module_info(&module)? { + let mut symbols = mi.symbols()?; + + while let Some(symbol) = symbols.next()? { + match symbol.parse() { + Ok(SymbolData::Procedure(proc)) => { + let noreturn = proc.flags.never; + + if noreturn { + let internal = proc.offset; + let offset = self + .translator + .internal_section_offset_to_virtual_offset(internal)?; + extra.noreturns.insert(offset); + } + } + Ok(SymbolData::Label(label)) => { + let internal = label.offset; + let offset = self + .translator + .internal_section_offset_to_virtual_offset(internal)?; + extra.labels.insert(offset); + } + _ => {} + } + } + } + } + + Ok(extra) + } +} + +#[derive(Default)] +struct ExtraDebugInfo { + /// Jump targets, typically from `switch` cases. + pub labels: BTreeSet, + + /// Entry offsets functions that do not return. + pub noreturns: BTreeSet, +} + +struct Translator<'data> { + base: Address, + address_map: AddressMap<'data>, + sections: Vec, +} + +impl<'data> Translator<'data> { + pub fn new<'a>(base: Address, pdb: &'a mut PDB<'data, Cursor<&'data [u8]>>) -> Result { + let address_map = pdb.address_map()?; + + let sections = pdb + .sections()? + .ok_or_else(|| anyhow::anyhow!("error reading section headers"))?; + + Ok(Self { + base, + address_map, + sections, + }) + } + + pub fn virtual_offset_to_file_offset(&self, offset: Offset) -> Result { + let internal = pdb::Rva(u32::try_from(offset.0)?) + .to_internal_offset(&self.address_map) + .ok_or_else(|| anyhow::anyhow!("could not map virtual offset to internal"))?; + + Ok(self.internal_section_offset_to_file_offset(internal)) + } + + pub fn internal_section_offset_to_file_offset( + &self, + internal: PdbInternalSectionOffset, + ) -> u32 { + let section_index = (internal.section - 1) as usize; + let section = self.sections[section_index]; + + let section_file_offset = section.pointer_to_raw_data; + section_file_offset + internal.offset + } + + pub fn internal_section_offset_to_virtual_offset( + &self, + internal: PdbInternalSectionOffset, + ) -> Result { + let rva = internal + .to_rva(&self.address_map) + .ok_or_else(|| anyhow::anyhow!("no virtual offset for internal section offset"))?; + + Ok(Offset(u64::from(rva.0))) + } +} diff --git a/src/agent/onefuzz-agent/src/config.rs b/src/agent/onefuzz-agent/src/config.rs index 3e50ab7ae6..d26d54e7f0 100644 --- a/src/agent/onefuzz-agent/src/config.rs +++ b/src/agent/onefuzz-agent/src/config.rs @@ -80,7 +80,7 @@ struct RawStaticConfig { } impl StaticConfig { - pub async fn new(data: &[u8]) -> Result { + pub async fn new(data: &[u8], machine_identity: Option) -> Result { let config: RawStaticConfig = serde_json::from_slice(data)?; let credentials = match config.client_credentials { @@ -104,7 +104,7 @@ impl StaticConfig { managed.into() } }; - let machine_identity = match config.machine_identity { + let machine_identity = match machine_identity.or(config.machine_identity) { Some(machine_identity) => machine_identity, None => MachineIdentity::from_metadata().await?, }; @@ -125,11 +125,14 @@ impl StaticConfig { Ok(config) } - pub async fn from_file(config_path: impl AsRef) -> Result { + pub async fn from_file( + config_path: impl AsRef, + machine_identity: Option, + ) -> Result { let config_path = config_path.as_ref(); let data = std::fs::read(config_path) .with_context(|| format!("unable to read config file: {}", config_path.display()))?; - Self::new(&data).await + Self::new(&data, machine_identity).await } pub fn from_env() -> Result { diff --git a/src/agent/onefuzz-agent/src/main.rs b/src/agent/onefuzz-agent/src/main.rs index 12428db2ba..38ca6ef621 100644 --- a/src/agent/onefuzz-agent/src/main.rs +++ b/src/agent/onefuzz-agent/src/main.rs @@ -170,8 +170,6 @@ fn run(opt: RunOpt) -> Result<()> { if opt.redirect_output.is_some() { return redirect(opt); } - let opt_machine_id = opt.machine_id; - let opt_machine_name = opt.machine_name.clone(); let rt = tokio::runtime::Runtime::new()?; let reset_lock = opt.reset_node_lock; let config = rt.block_on(load_config(opt)); @@ -184,15 +182,6 @@ fn run(opt: RunOpt) -> Result<()> { let config = config?; - let config = StaticConfig { - machine_identity: MachineIdentity { - machine_id: opt_machine_id.unwrap_or(config.machine_identity.machine_id), - machine_name: opt_machine_name.unwrap_or(config.machine_identity.machine_name), - ..config.machine_identity - }, - ..config - }; - if reset_lock { done::remove_done_lock(config.machine_identity.machine_id)?; } else if done::is_agent_done(config.machine_identity.machine_id)? { @@ -218,10 +207,18 @@ fn run(opt: RunOpt) -> Result<()> { } async fn load_config(opt: RunOpt) -> Result { - info!("loading supervisor agent config"); + info!("loading supervisor agent config: {:?}", opt); + let opt_machine_id = opt.machine_id; + let opt_machine_name = opt.machine_name.clone(); + + let machine_identity = opt_machine_id.map(|machine_id| MachineIdentity { + machine_id, + machine_name: opt_machine_name.unwrap_or(format!("{}", machine_id)), + scaleset_name: None, + }); let config = match &opt.config_path { - Some(config_path) => StaticConfig::from_file(config_path).await?, + Some(config_path) => StaticConfig::from_file(config_path, machine_identity).await?, None => StaticConfig::from_env()?, }; diff --git a/src/agent/onefuzz/src/auth.rs b/src/agent/onefuzz/src/auth.rs index d25a3807f9..2def380c3b 100644 --- a/src/agent/onefuzz/src/auth.rs +++ b/src/agent/onefuzz/src/auth.rs @@ -113,9 +113,11 @@ impl ClientCredentials { pub async fn access_token(&self) -> Result { let (authority, scope) = { let url = Url::parse(&self.resource.clone())?; - let host = url.host_str().ok_or_else(|| { + let port = url.port().map(|p| format!(":{}", p)).unwrap_or_default(); + let host_name = url.host_str().ok_or_else(|| { anyhow::format_err!("resource URL does not have a host string: {}", url) })?; + let host = format!("{}{}", host_name, port); if let Some(domain) = &self.multi_tenant_domain { let instance: Vec<&str> = host.split('.').collect(); ( diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index f7659356d7..a7b579c57f 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -14,6 +14,7 @@ from enum import Enum from shutil import which from typing import Callable, Dict, List, Optional, Tuple, Type, TypeVar +from urllib.parse import urlparse from uuid import UUID import semver @@ -1268,6 +1269,16 @@ def get_config(self, pool_name: primitives.PoolName) -> models.AgentConfig: if pool.config is None: raise Exception("Missing AgentConfig in response") + config = pool.config + if not pool.managed: + config.client_credentials = models.ClientCredentials( # nosec + client_id=uuid.UUID(int=0), + client_secret="", + resource=self.onefuzz._backend.config.endpoint, + tenant=urlparse(self.onefuzz._backend.config.authority).path.strip("/"), + multi_tenant_domain=self.onefuzz._backend.config.tenant_domain, + ) + return pool.config def shutdown(self, name: str, *, now: bool = False) -> responses.BoolResult: diff --git a/src/deployment/azuredeploy.bicep b/src/deployment/azuredeploy.bicep index 20568e5eb7..14123b8182 100644 --- a/src/deployment/azuredeploy.bicep +++ b/src/deployment/azuredeploy.bicep @@ -117,19 +117,6 @@ resource keyVault 'Microsoft.KeyVault/vaults@2021-10-01' = { ] } } - { - objectId: netFunction.outputs.principalId - tenantId: tenantId - permissions: { - secrets: [ - 'get' - 'list' - 'set' - 'delete' - ] - } - } - ] tenantId: tenantId } @@ -192,21 +179,6 @@ resource roleAssignments 'Microsoft.Authorization/roleAssignments@2020-10-01-pre ] }] -// try to make role assignments to deploy as late as possible in order to have principalId ready -resource roleAssignmentsNet 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = [for r in roleAssignmentsParams: { - name: guid('${resourceGroup().id}${r.suffix}-1f-net') - properties: { - roleDefinitionId: '/subscriptions/${subscription().subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/${r.role}' - principalId: netFunction.outputs.principalId - } - dependsOn: [ - eventGrid - keyVault - serverFarm - featureFlags - ] -}] - // try to make role assignments to deploy as late as possible in order to have principalId ready resource readBlobUserAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { name: guid('${resourceGroup().id}-user_managed_idenity_read_blob') @@ -250,27 +222,6 @@ module function 'bicep-templates/function.bicep' = { } } -module netFunction 'bicep-templates/function.bicep' = { - name: 'netFunction' - params: { - linux_fx_version: 'DOTNET-ISOLATED|7.0' - name: '${name}-net' - - app_logs_sas_url: storage.outputs.FuncSasUrlBlobAppLogs - app_func_audiences: app_func_audiences - app_func_issuer: app_func_issuer - client_id: clientId - diagnostics_log_level: diagnosticsLogLevel - location: location - log_retention: log_retention - owner: owner - server_farm_id: serverFarm.outputs.id - - use_windows: true - enable_remote_debugging: enable_remote_debugging - } -} - module functionSettings 'bicep-templates/function-settings.bicep' = { name: 'functionSettings' params: { @@ -291,118 +242,12 @@ module functionSettings 'bicep-templates/function-settings.bicep' = { multi_tenant_domain: multi_tenant_domain enable_profiler: enable_profiler app_config_endpoint: featureFlags.outputs.AppConfigEndpoint - functions_disabled: '0' - agent_function_names: [ - 'AgentCanSchedule' //0 - 'AgentCommands' //1 - 'AgentEvents' //2 - 'AgentRegistration' //3 - 'Containers' //4 - 'Download' //5 - 'Info' //6 - 'InstanceConfig' //7 - 'Jobs' //8 - 'JobTemplates' //9 - 'JobTemplatesManage' //10 - 'Negotiate' //11 - 'Node' //12 - 'NodeAddSshKey' //13 - 'Notifications' //14 - 'Pool' //15 - 'Proxy' //16 - 'QueueFileChanges' //17 - 'QueueNodeHeartbeat' //18 - 'QueueProxyUpdate' //19 - 'QueueSignalrEvents' //20 - 'QueueTaskHeartbeat' //21 - 'QueueUpdates' //22 - 'QueueWebhooks' //23 - 'ReproVms' //24 - 'Scaleset' //25 - 'Tasks' //26 - 'TimerDaily' //27 - 'TimerProxy' //28 - 'TimerRepro' //29 - 'TimerRetention' //30 - 'TimerTasks' //31 - 'TimerWorkers' //32 - 'Tools' //33 - 'Webhooks' //34 - 'WebhooksLogs' //35 - 'WebhooksPing' //36 - ] } dependsOn: [ function ] } -module netFunctionSettings 'bicep-templates/function-settings.bicep' = { - name: 'netFunctionSettings' - params: { - owner: owner - name: '${name}-net' - functions_worker_runtime: 'dotnet-isolated' - functions_extension_version: '~4' - instance_name: name - app_insights_app_id: operationalInsights.outputs.appInsightsAppId - app_insights_key: operationalInsights.outputs.appInsightsInstrumentationKey - client_secret: clientSecret - signal_r_connection_string: signalR.outputs.connectionString - func_sas_url: storage.outputs.FuncSasUrl - func_storage_resource_id: storage.outputs.FuncId - fuzz_storage_resource_id: storage.outputs.FuzzId - keyvault_name: keyVaultName - monitor_account_name: operationalInsights.outputs.monitorAccountName - multi_tenant_domain: multi_tenant_domain - enable_profiler: enable_profiler - app_config_endpoint: featureFlags.outputs.AppConfigEndpoint - functions_disabled: '1' - agent_function_names: [ - 'AgentCanSchedule' //0 - 'AgentCommands' //1 - 'AgentEvents' //2 - 'AgentRegistration' //3 - 'Containers' //4 - 'Download' //5 - 'Info' //6 - 'InstanceConfig' //7 - 'Jobs' //8 - 'JobTemplates' //9 - 'JobTemplatesManage' //10 - 'Negotiate' //11 - 'Node' //12 - 'NodeAddSshKey' //13 - 'Notifications' //14 - 'Pool' //15 - 'Proxy' //16 - 'QueueFileChanges' //17 - 'QueueNodeHeartbeat' //18 - 'QueueProxyUpdate' //19 - 'QueueSignalrEvents' //20 - 'QueueTaskHeartbeat' //21 - 'QueueUpdates' //22 - 'QueueWebhooks' //23 - 'ReproVms' //24 - 'Scaleset' //25 - 'Tasks' //26 - 'TimerDaily' //27 - 'TimerProxy' //28 - 'TimerRepro' //29 - 'TimerRetention' //30 - 'TimerTasks' //31 - 'TimerWorkers' //32 - 'Tools' //33 - 'Webhooks' //34 - 'WebhookLogs' //35 - 'WebhookPing' //36 - ] - } - dependsOn: [ - netFunction - ] -} - output fuzz_storage string = storage.outputs.FuzzId output fuzz_name string = storage.outputs.FuzzName output fuzz_key string = storage.outputs.FuzzKey diff --git a/src/deployment/bicep-templates/function-settings.bicep b/src/deployment/bicep-templates/function-settings.bicep index 739681f26a..2f65f8d5f4 100644 --- a/src/deployment/bicep-templates/function-settings.bicep +++ b/src/deployment/bicep-templates/function-settings.bicep @@ -28,27 +28,14 @@ param monitor_account_name string param functions_worker_runtime string param functions_extension_version string -param agent_function_names array -param functions_disabled string - param enable_profiler bool -var disabledFunctionName = 'disabledFunctions-${name}' - var telemetry = 'd7a73cf4-5a1a-4030-85e1-e5b25867e45a' resource function 'Microsoft.Web/sites@2021-02-01' existing = { name: name } -module disabledFunctions 'function-settings-disabled-apps.bicep' = { - name: disabledFunctionName - params:{ - functions_disabled_setting: functions_disabled - allFunctions: agent_function_names - } -} - var enable_profilers = enable_profiler ? { APPINSIGHTS_PROFILERFEATURE_VERSION : '1.0.0' DiagnosticServices_EXTENSION_VERSION: '~3' @@ -79,5 +66,5 @@ resource functionSettings 'Microsoft.Web/sites/config@2021-03-01' = { ONEFUZZ_KEYVAULT: keyvault_name ONEFUZZ_OWNER: owner ONEFUZZ_CLIENT_SECRET: client_secret - }, disabledFunctions.outputs.appSettings, enable_profilers) + }, enable_profilers) } diff --git a/src/deployment/deploy.py b/src/deployment/deploy.py index cafa7658a0..1618da6122 100644 --- a/src/deployment/deploy.py +++ b/src/deployment/deploy.py @@ -99,8 +99,6 @@ "specifying for this argument and retry." ) -DOTNET_APPLICATION_SUFFIX = "-net" - logger = logging.getLogger("deploy") @@ -301,49 +299,25 @@ def create_password(self, object_id: UUID) -> Tuple[str, str]: "cli_password", object_id, self.get_subscription_id() ) - def get_instance_urls(self) -> List[str]: + def get_instance_url(self) -> str: # The url to access the instance # This also represents the legacy identifier_uris of the application # registration if self.multi_tenant_domain: - return [ - "https://%s/%s" % (self.multi_tenant_domain, name) - for name in [ - self.application_name, - self.application_name + DOTNET_APPLICATION_SUFFIX, - ] - ] + return "https://%s/%s" % (self.multi_tenant_domain, self.application_name) else: - return [ - "https://%s.azurewebsites.net" % name - for name in [ - self.application_name, - self.application_name + DOTNET_APPLICATION_SUFFIX, - ] - ] + return "https://%s.azurewebsites.net" % self.application_name - def get_identifier_urls(self) -> List[str]: + def get_identifier_url(self) -> str: # This is used to identify the application registration via the # identifier_uris field. Depending on the environment this value needs # to be from an approved domain The format of this value is derived # from the default value proposed by azure when creating an application # registration api://{guid}/... if self.multi_tenant_domain: - return [ - "api://%s/%s" % (self.multi_tenant_domain, name) - for name in [ - self.application_name, - self.application_name + DOTNET_APPLICATION_SUFFIX, - ] - ] + return "api://%s/%s" % (self.multi_tenant_domain, self.application_name) else: - return [ - "api://%s.azurewebsites.net" % name - for name in [ - self.application_name, - self.application_name + DOTNET_APPLICATION_SUFFIX, - ] - ] + return "api://%s.azurewebsites.net" % self.application_name def get_signin_audience(self) -> str: # https://docs.microsoft.com/en-us/azure/active-directory/develop/supported-accounts-validation @@ -514,7 +488,7 @@ def update_existing_app_registration( # find any identifier URIs that need updating identifier_uris: List[str] = app["identifierUris"] updated_identifier_uris = list( - set(identifier_uris) | set(self.get_identifier_urls()) + set(identifier_uris) | set([self.get_identifier_url()]) ) if len(updated_identifier_uris) > len(identifier_uris): update_properties["identifierUris"] = updated_identifier_uris @@ -561,7 +535,7 @@ def create_new_app_registration( params = { "displayName": self.application_name, - "identifierUris": self.get_identifier_urls(), + "identifierUris": [self.get_identifier_url()], "signInAudience": self.get_signin_audience(), "appRoles": app_roles, "api": { @@ -583,10 +557,7 @@ def create_new_app_registration( "enableAccessTokenIssuance": False, "enableIdTokenIssuance": True, }, - "redirectUris": [ - f"{url}/.auth/login/aad/callback" - for url in self.get_instance_urls() - ], + "redirectUris": [f"{self.get_instance_url()}/.auth/login/aad/callback"], }, "requiredResourceAccess": [ { @@ -662,8 +633,8 @@ def deploy_template(self) -> None: "%Y-%m-%dT%H:%M:%SZ" ) - app_func_audiences = self.get_identifier_urls().copy() - app_func_audiences.extend(self.get_instance_urls()) + app_func_audiences = [self.get_identifier_url()] + app_func_audiences.extend([self.get_instance_url()]) if self.multi_tenant_domain: # clear the value in the Issuer Url field: @@ -1135,45 +1106,6 @@ def deploy_app(self) -> None: if error is not None: raise error - def deploy_dotnet_app(self) -> None: - logger.info("deploying function app %s ", self.app_zip) - with tempfile.TemporaryDirectory() as tmpdirname: - with zipfile.ZipFile(self.app_zip, "r") as zip_ref: - func = shutil.which("func") - assert func is not None - - zip_ref.extractall(tmpdirname) - error: Optional[subprocess.CalledProcessError] = None - max_tries = 5 - for i in range(max_tries): - try: - subprocess.check_output( - [ - func, - "azure", - "functionapp", - "publish", - self.application_name + DOTNET_APPLICATION_SUFFIX, - "--no-build", - "--dotnet-version", - "7.0", - ], - env=dict(os.environ, CLI_DEBUG="1"), - cwd=tmpdirname, - ) - return - except subprocess.CalledProcessError as err: - error = err - if i + 1 < max_tries: - logger.debug("func failure error: %s", err) - logger.warning( - "function failed to deploy, waiting 60 " - "seconds and trying again" - ) - time.sleep(60) - if error is not None: - raise error - def update_registration(self) -> None: if not self.create_registration: return @@ -1241,7 +1173,6 @@ def main() -> None: ("instance-specific-setup", Client.upload_instance_setup), ("third-party", Client.upload_third_party), ("api", Client.deploy_app), - ("dotnet-api", Client.deploy_dotnet_app), ("export_appinsights", Client.add_log_export), ("update_registration", Client.update_registration), ] diff --git a/src/deployment/deploylib/registration.py b/src/deployment/deploylib/registration.py index 641ae62203..e5c5a42b21 100644 --- a/src/deployment/deploylib/registration.py +++ b/src/deployment/deploylib/registration.py @@ -861,10 +861,13 @@ def main() -> None: "--registration_name", help="the name of the cli registration" ) register_app_parser = subparsers.add_parser("register_app", parents=[parent_parser]) - register_app_parser.add_argument("--app_id", help="the application id to register") + register_app_parser.add_argument( + "--app_id", help="the application id to register", required=True + ) register_app_parser.add_argument( "--role", help=f"the role of the application to register. Valid values: {', '.join([member.value for member in OnefuzzAppRole])}", + required=True, ) args = parser.parse_args() diff --git a/src/proxy-manager/Cargo.lock b/src/proxy-manager/Cargo.lock index 96d0d3919e..c69ddc111f 100644 --- a/src/proxy-manager/Cargo.lock +++ b/src/proxy-manager/Cargo.lock @@ -97,6 +97,15 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -621,14 +630,14 @@ checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" [[package]] name = "mio" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" +checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -791,7 +800,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.36.1", ] [[package]] @@ -1018,6 +1027,7 @@ dependencies = [ "log", "onefuzz-telemetry", "reqwest", + "thiserror", ] [[package]] @@ -1210,6 +1220,7 @@ dependencies = [ "async-trait", "backoff", "base64", + "bincode", "bytes", "derivative", "flume", @@ -1323,9 +1334,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.20.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57aec3cfa4c296db7255446efb4928a6be304b431a806216105542a67b6ca82e" +checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" dependencies = [ "autocfg", "bytes", @@ -1333,13 +1344,12 @@ dependencies = [ "memchr", "mio", "num_cpus", - "once_cell", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -1646,43 +1656,100 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_msvc", + "windows_aarch64_msvc 0.36.1", + "windows_i686_gnu 0.36.1", + "windows_i686_msvc 0.36.1", + "windows_x86_64_gnu 0.36.1", + "windows_x86_64_msvc 0.36.1", ] +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.0", + "windows_i686_gnu 0.42.0", + "windows_i686_msvc 0.42.0", + "windows_x86_64_gnu 0.42.0", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" + [[package]] name = "windows_aarch64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" + [[package]] name = "windows_i686_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" +[[package]] +name = "windows_i686_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" + [[package]] name = "windows_i686_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" +[[package]] +name = "windows_i686_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" + [[package]] name = "windows_x86_64_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" + [[package]] name = "windows_x86_64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" + [[package]] name = "winreg" version = "0.10.1" diff --git a/src/proxy-manager/Cargo.toml b/src/proxy-manager/Cargo.toml index 43860a94a3..865d9aee0b 100644 --- a/src/proxy-manager/Cargo.toml +++ b/src/proxy-manager/Cargo.toml @@ -16,7 +16,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" storage-queue = { path = "../agent/storage-queue" } thiserror = "1.0" -tokio = { version = "1.20", features = ["macros", "rt-multi-thread", "fs", "process"] } +tokio = { version = "1.23", features = ["macros", "rt-multi-thread", "fs", "process"] } url = { version = "2.3", features = ["serde"] } reqwest-retry = { path = "../agent/reqwest-retry"} onefuzz-telemetry = { path = "../agent/onefuzz-telemetry" } diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index 858dd18c3d..60fa68cc12 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -328,6 +328,9 @@ class SyncedDir(BaseModel): class ClientCredentials(BaseModel): client_id: UUID client_secret: str + resource: str + tenant: str + multi_tenant_domain: Optional[str] class AgentConfig(BaseModel): diff --git a/src/runtime-tools/linux/set-env.sh b/src/runtime-tools/linux/set-env.sh new file mode 100644 index 0000000000..33ec3ddfd7 --- /dev/null +++ b/src/runtime-tools/linux/set-env.sh @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +export DOTNET_ROOT=/onefuzz/tools/dotnet +export DOTNET_CLI_HOME="$DOTNET_ROOT" +export LLVM_SYMBOLIZER_PATH=/onefuzz/bin/llvm-symbolizer +export RUST_LOG = "info" \ No newline at end of file diff --git a/src/runtime-tools/win64/set-env.ps1 b/src/runtime-tools/win64/set-env.ps1 new file mode 100644 index 0000000000..20546bab07 --- /dev/null +++ b/src/runtime-tools/win64/set-env.ps1 @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +$env:Path += ";C:\Program Files (x86)\Windows Kits\10\Debuggers\x64\;C:\onefuzz\win64;C:\onefuzz\tools\win64;C:\onefuzz\tools\win64\radamsa;$env:ProgramFiles\LLVM\bin" +$env:LLVM_SYMBOLIZER_PATH = "C:\Program Files\LLVM\bin\llvm-symbolizer.exe" +$env:RUST_LOG = "info"