Skip to content

Commit b75fab2

Browse files
gleocadiegleocadie
authored
Download and Copy libdatadog binaries for Data-Pipeline integration (#6777)
1 parent c73ea92 commit b75fab2

File tree

7 files changed

+254
-76
lines changed

7 files changed

+254
-76
lines changed

‎build/cmake/FindLibdatadog.cmake‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,22 @@ include(FetchContent)
66

77
set(LIBDATADOG_VERSION "v16.0.3" CACHE STRING "libdatadog version")
88

9-
if (CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL arm64)
10-
if (DEFINED ENV{IsAlpine} AND "$ENV{IsAlpine}" MATCHES "true")
9+
if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")
10+
if (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
11+
set(SHA256_LIBDATADOG "2d7933e09dc39706e9c99c7edcff5c60f7567ea2777157596de828f62f39035b" CACHE STRING "libdatadog sha256")
12+
set(FILE_TO_DOWNLOAD libdatadog-aarch64-apple-darwin.tar.gz)
13+
elseif (DEFINED ENV{IsAlpine} AND "$ENV{IsAlpine}" MATCHES "true")
1114
set(SHA256_LIBDATADOG "dd08d3a4dbbd765392121d27b790d7818e80dd28500b554db16e9186b1025ba9" CACHE STRING "libdatadog sha256")
1215
set(FILE_TO_DOWNLOAD libdatadog-aarch64-alpine-linux-musl.tar.gz)
1316
else()
1417
set(SHA256_LIBDATADOG "decc01a2e0f732cabcc56594429a3dbc13678070e07f24891555dcc02df2e516" CACHE STRING "libdatadog sha256")
1518
set(FILE_TO_DOWNLOAD libdatadog-aarch64-unknown-linux-gnu.tar.gz)
1619
endif()
1720
else()
18-
if (DEFINED ENV{IsAlpine} AND "$ENV{IsAlpine}" MATCHES "true")
21+
if (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
22+
set(SHA256_LIBDATADOG "ced5db61e0ca8e974b9d59b0b6833c28e19445a3e4ec3c548fda965806c17560" CACHE STRING "libdatadog sha256")
23+
set(FILE_TO_DOWNLOAD libdatadog-x86_64-apple-darwin.tar.gz)
24+
elseif (DEFINED ENV{IsAlpine} AND "$ENV{IsAlpine}" MATCHES "true")
1925
set(SHA256_LIBDATADOG "8e09afd3cfb5ace85501f37b4bd6378299ebbf71189ccc2173169998b75b4b56" CACHE STRING "libdatadog sha256")
2026
set(FILE_TO_DOWNLOAD libdatadog-${CMAKE_SYSTEM_PROCESSOR}-alpine-linux-musl.tar.gz)
2127
else()
@@ -24,16 +30,16 @@ else()
2430
endif()
2531
endif()
2632

27-
FetchContent_Declare(libdatadog-${LIBDATADOG_VERSION}
33+
FetchContent_Declare(libdatadog-install
2834
URL https://github.com/DataDog/libdatadog/releases/download/${LIBDATADOG_VERSION}/${FILE_TO_DOWNLOAD}
2935
URL_HASH SHA256=${SHA256_LIBDATADOG}
30-
SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/libdatadog-${LIBDATADOG_VERSION}
36+
SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/libdatadog-install
3137
)
32-
if(NOT libdatadog-${LIBDATADOG_VERSION}_POPULATED)
33-
FetchContent_Populate(libdatadog-${LIBDATADOG_VERSION})
38+
if(NOT libdatadog-install_POPULATED)
39+
FetchContent_Populate(libdatadog-install)
3440
endif()
3541

36-
set(LIBDATADOG_BASE_DIR ${libdatadog-${LIBDATADOG_VERSION}_SOURCE_DIR})
42+
set(LIBDATADOG_BASE_DIR ${libdatadog-install_SOURCE_DIR})
3743

3844
add_library(libdatadog-lib SHARED IMPORTED)
3945

@@ -42,7 +48,7 @@ set_target_properties(libdatadog-lib PROPERTIES
4248
IMPORTED_LOCATION ${LIBDATADOG_BASE_DIR}/lib/libdatadog_profiling.so
4349
)
4450

45-
add_dependencies(libdatadog-lib libdatadog-${LIBDATADOG_VERSION})
51+
add_dependencies(libdatadog-lib libdatadog-install)
4652

4753
# Override target_link_libraries
4854
function(target_link_libraries target)

‎tracer/build/_build/Build.Profiler.Steps.cs‎

Lines changed: 1 addition & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -45,74 +45,8 @@ partial class Build
4545
.OnlyWhenStatic(() => IsWin)
4646
.Executes(async () =>
4747
{
48-
var vcpkg = await GetVcpkg();
48+
var vcpkg = ToolResolver.GetLocalTool(await GetVcpkg());
4949
vcpkg("integrate install");
50-
51-
async Task<Tool> GetVcpkg()
52-
{
53-
var vcpkgFilePath = string.Empty;
54-
55-
try
56-
{
57-
vcpkgFilePath = ToolPathResolver.GetPathExecutable("vcpkg.exe");
58-
}
59-
catch (ArgumentException)
60-
{ }
61-
62-
if (File.Exists(vcpkgFilePath))
63-
{
64-
return ToolResolver.GetLocalTool(vcpkgFilePath);
65-
}
66-
67-
// Check if already downloaded
68-
var vcpkgRoot = RootDirectory / "artifacts" / "bin" / "vcpkg";
69-
var vcpkgExecPath = vcpkgRoot / "vcpkg.exe";
70-
71-
if (File.Exists(vcpkgExecPath))
72-
{
73-
return ToolResolver.GetLocalTool($"{vcpkgExecPath}");
74-
}
75-
76-
await DownloadAndExtractVcpkg(vcpkgRoot);
77-
Cmd.Value(arguments: $"cmd /c {vcpkgRoot / "bootstrap-vcpkg.bat"}");
78-
return ToolResolver.GetLocalTool($"{vcpkgRoot / "vcpkg.exe"}");
79-
}
80-
81-
async Task DownloadAndExtractVcpkg(AbsolutePath destinationFolder)
82-
{
83-
var nbTries = 0;
84-
var keepTrying = true;
85-
var vcpkgZip = TempDirectory / "vcpkg.zip";
86-
using var client = new HttpClient();
87-
const string vcpkgVersion = "2024.11.16";
88-
while (keepTrying)
89-
{
90-
nbTries++;
91-
try
92-
{
93-
var response = await client.GetAsync($"https://github.com/microsoft/vcpkg/archive/refs/tags/{vcpkgVersion}.zip");
94-
response.EnsureSuccessStatusCode();
95-
await using var stream = await response.Content.ReadAsStreamAsync();
96-
await using var file = File.Create(vcpkgZip);
97-
await stream.CopyToAsync(file);
98-
keepTrying = false;
99-
}
100-
catch (HttpRequestException)
101-
{
102-
if (nbTries > 3)
103-
{
104-
throw;
105-
}
106-
}
107-
}
108-
109-
EnsureExistingParentDirectory(destinationFolder);
110-
var parentFolder = destinationFolder.Parent;
111-
112-
CompressionTasks.UncompressZip(vcpkgZip, parentFolder);
113-
114-
RenameDirectory(parentFolder / $"vcpkg-{vcpkgVersion}", destinationFolder.Name);
115-
}
11650
});
11751

11852
Target CompileProfilerNativeSrcWindows => _ => _

‎tracer/build/_build/Build.Steps.cs‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -552,6 +552,73 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo
552552
}
553553
});
554554

555+
Target DownloadLibDatadog => _ => _
556+
.Unlisted()
557+
.After(CreateRequiredDirectories)
558+
.Executes(async () =>
559+
{
560+
if (IsLinux || IsOsx)
561+
{
562+
if (!Directory.Exists(NativeBuildDirectory))
563+
{
564+
CMake.Value(
565+
arguments: $"-DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang -B {NativeBuildDirectory} -S {RootDirectory}");
566+
}
567+
// In CI, the folder might already exist. Which means that CMake was already configured
568+
// and libdatadog artifact (correct one) was already downloaded.
569+
// So just reuse it.
570+
571+
}
572+
else if (IsWin)
573+
{
574+
var vcpkgExePath = await GetVcpkg();
575+
var vcpkgRoot = Environment.GetEnvironmentVariable("VCPKG_ROOT") ?? Directory.GetParent(vcpkgExePath).FullName;
576+
var vcpkg = ToolResolver.GetLocalTool(vcpkgExePath);
577+
foreach (var arch in new[] { MSBuildTargetPlatform.x64, MSBuildTargetPlatform.x86 })
578+
{
579+
// This big line is the same generated by VS when installing libdatadog while building the profiler
580+
vcpkg($"install --x-wait-for-lock --triplet \"{arch}-windows\" --vcpkg-root \"{vcpkgRoot}\" \"--x-manifest-root={RootDirectory}\" \"--x-install-root={BuildArtifactsDirectory}\\deps\\vcpkg\\{arch}-windows\" --downloads-root {BuildArtifactsDirectory}\\obj\\vcpkg\\downloads --x-packages-root {BuildArtifactsDirectory}\\obj\\vcpkg\\packages --x-buildtrees-root {BuildArtifactsDirectory}\\obj\\vcpkg/buildtrees --clean-after-build");
581+
}
582+
}
583+
});
584+
585+
Target CopyLibDatadog => _ => _
586+
.Unlisted()
587+
.After(DownloadLibDatadog)
588+
.Executes(() =>
589+
{
590+
if (IsWin)
591+
{
592+
var vcpkgDepsFolder = BuildArtifactsDirectory / "deps" / "vcpkg";
593+
foreach (var architecture in new[] { MSBuildTargetPlatform.x64, MSBuildTargetPlatform.x86 })
594+
{
595+
var source = vcpkgDepsFolder / $"{architecture}-windows" / $"{architecture}-windows";
596+
if (BuildConfiguration == Configuration.Debug)
597+
{
598+
source /= "debug";
599+
}
600+
601+
var dllFile = source / "bin" / "datadog_profiling_ffi.dll";
602+
var dest = MonitoringHomeDirectory / $"win-{architecture}";
603+
CopyFileToDirectory(dllFile, dest, FileExistsPolicy.Overwrite);
604+
605+
var pdbFile = source / "bin" / "datadog_profiling_ffi.pdb";
606+
dest = SymbolsDirectory / $"win-{architecture}";
607+
CopyFileToDirectory(pdbFile, dest, FileExistsPolicy.Overwrite);
608+
}
609+
}
610+
else if (IsLinux || IsOsx)
611+
{
612+
var (destArch, ext) = GetUnixArchitectureAndExtension();
613+
614+
var libdatadogFileName = $"libdatadog_profiling.{ext}";
615+
616+
var source = NativeBuildDirectory / "libdatadog-install" / "lib" / libdatadogFileName;
617+
var dest = MonitoringHomeDirectory / destArch;
618+
CopyFileToDirectory(source, dest, FileExistsPolicy.Overwrite);
619+
}
620+
});
621+
555622
Target CopyNativeFilesForAppSecUnitTests => _ => _
556623
.Unlisted()
557624
.After(Clean)
@@ -2746,4 +2813,71 @@ private void DotnetBuild(
27462813
.SetProcessArgumentConfigurator(arg => arg.Add("/nowarn:NU1701")) //nowarn:NU1701 - Package 'x' was restored using '.NETFramework,Version=v4.6.1' instead of the project target framework '.NETCoreApp,Version=v2.1'.
27472814
.CombineWith(projPaths, (settings, projPath) => settings.SetProjectFile(projPath)));
27482815
}
2816+
2817+
2818+
private async Task<string> GetVcpkg()
2819+
{
2820+
var vcpkgFilePath = string.Empty;
2821+
2822+
try
2823+
{
2824+
vcpkgFilePath = ToolPathResolver.GetPathExecutable("vcpkg.exe");
2825+
}
2826+
catch (ArgumentException)
2827+
{ }
2828+
2829+
if (File.Exists(vcpkgFilePath))
2830+
{
2831+
return vcpkgFilePath;
2832+
}
2833+
2834+
// Check if already downloaded
2835+
var vcpkgRoot = RootDirectory / "artifacts" / "bin" / "vcpkg";
2836+
var vcpkgExecPath = vcpkgRoot / "vcpkg.exe";
2837+
2838+
if (File.Exists(vcpkgExecPath))
2839+
{
2840+
return $"{vcpkgExecPath}";
2841+
}
2842+
2843+
await DownloadAndExtractVcpkg(vcpkgRoot);
2844+
Cmd.Value(arguments: $"cmd /c {vcpkgRoot / "bootstrap-vcpkg.bat"}");
2845+
return $"{vcpkgRoot / "vcpkg.exe"}";
2846+
}
2847+
2848+
private async Task DownloadAndExtractVcpkg(AbsolutePath destinationFolder)
2849+
{
2850+
var nbTries = 0;
2851+
var keepTrying = true;
2852+
var vcpkgZip = TempDirectory / "vcpkg.zip";
2853+
using var client = new HttpClient();
2854+
const string vcpkgVersion = "2024.11.16";
2855+
while (keepTrying)
2856+
{
2857+
nbTries++;
2858+
try
2859+
{
2860+
var response = await client.GetAsync($"https://github.com/microsoft/vcpkg/archive/refs/tags/{vcpkgVersion}.zip");
2861+
response.EnsureSuccessStatusCode();
2862+
await using var stream = await response.Content.ReadAsStreamAsync();
2863+
await using var file = File.Create(vcpkgZip);
2864+
await stream.CopyToAsync(file);
2865+
keepTrying = false;
2866+
}
2867+
catch (HttpRequestException)
2868+
{
2869+
if (nbTries > 3)
2870+
{
2871+
throw;
2872+
}
2873+
}
2874+
}
2875+
2876+
EnsureExistingParentDirectory(destinationFolder);
2877+
var parentFolder = destinationFolder.Parent;
2878+
2879+
CompressionTasks.UncompressZip(vcpkgZip, parentFolder);
2880+
2881+
RenameDirectory(parentFolder / $"vcpkg-{vcpkgVersion}", destinationFolder.Name);
2882+
}
27492883
}

‎tracer/build/_build/Build.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,8 @@ void DeleteReparsePoints(string path)
205205
.DependsOn(PublishManagedTracer)
206206
.DependsOn(DownloadLibDdwaf)
207207
.DependsOn(CopyLibDdwaf)
208+
.DependsOn(DownloadLibDatadog)
209+
.DependsOn(CopyLibDatadog)
208210
.DependsOn(CreateMissingNullabilityFile)
209211
.DependsOn(CreateTrimmingFile)
210212
.DependsOn(RegenerateSolutions);
@@ -219,6 +221,8 @@ void DeleteReparsePoints(string path)
219221
.DependsOn(PublishManagedTracerR2R)
220222
.DependsOn(DownloadLibDdwaf)
221223
.DependsOn(CopyLibDdwaf)
224+
.DependsOn(DownloadLibDatadog)
225+
.DependsOn(CopyLibDatadog)
222226
.DependsOn(CreateMissingNullabilityFile)
223227
.DependsOn(CreateTrimmingFile);
224228

‎tracer/src/Datadog.Trace/LibDatadog/NativeInterop.cs‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
// <copyright file="NativeInterop.cs" company="Datadog">
2+
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
3+
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
4+
// </copyright>
5+
6+
#nullable enable
7+
8+
using System;
9+
using System.Runtime.InteropServices;
10+
11+
namespace Datadog.Trace.LibDatadog;
12+
13+
internal class NativeInterop
14+
{
15+
private const string DllName = "LibDatadog";
16+
17+
internal static class Exporter
18+
{
19+
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_new")]
20+
// internal static extern ErrorHandle Create(out IntPtr outHandle, SafeHandle config);
21+
22+
[DllImport(DllName, EntryPoint = "ddog_trace_exporter_error_free")]
23+
internal static extern void ReleaseError(IntPtr error);
24+
25+
[DllImport(DllName, EntryPoint = "ddog_trace_exporter_free")]
26+
internal static extern void Release(IntPtr handle);
27+
28+
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_send")]
29+
// internal static extern ErrorHandle Send(SafeHandle handle, ByteSlice trace, UIntPtr traceCount, ref IntPtr response);
30+
}
31+
32+
internal static class Config
33+
{
34+
[DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_new")]
35+
internal static extern void Create(out IntPtr outHandle);
36+
37+
[DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_free")]
38+
internal static extern void Release(IntPtr handle);
39+
40+
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_url")]
41+
// internal static extern ErrorHandle SetUrl(SafeHandle config, CharSlice url);
42+
43+
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_tracer_version")]
44+
// internal static extern ErrorHandle SetTracerVersion(SafeHandle config, CharSlice version);
45+
46+
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_language")]
47+
// internal static extern ErrorHandle SetLanguage(SafeHandle config, CharSlice lang);
48+
49+
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_lang_version")]
50+
// internal static extern ErrorHandle SetLanguageVersion(SafeHandle config, CharSlice version);
51+
52+
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_lang_interpreter")]
53+
// internal static extern ErrorHandle SetInterperter(SafeHandle config, CharSlice interpreter);
54+
55+
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_hostname")]
56+
// internal static extern ErrorHandle SetHostname(SafeHandle config, CharSlice hostname);
57+
58+
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_env")]
59+
// internal static extern ErrorHandle SetEnvironment(SafeHandle config, CharSlice env);
60+
61+
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_version")]
62+
// internal static extern ErrorHandle SetVersion(SafeHandle config, CharSlice version);
63+
64+
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_service")]
65+
// internal static extern ErrorHandle SetService(SafeHandle config, CharSlice service);
66+
}
67+
}

0 commit comments

Comments
 (0)