From 2593e457006bc4cb07c17ba3971cdf85d0cabdaf Mon Sep 17 00:00:00 2001 From: Melissa Hale Date: Thu, 14 Dec 2023 21:41:55 -0600 Subject: [PATCH 1/2] datadog guide --- src/data/sidebar.ts | 3 +- src/docs/tutorials/set-up-a-datadog-agent.md | 187 +++++++++++++++++++ 2 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 src/docs/tutorials/set-up-a-datadog-agent.md diff --git a/src/data/sidebar.ts b/src/data/sidebar.ts index c11e7b257..7786879ff 100644 --- a/src/data/sidebar.ts +++ b/src/data/sidebar.ts @@ -127,10 +127,11 @@ export const sidebarContent: ISidebarContent = [ ] }, { - subTitle: "Technical Tutorial", + subTitle: "Technical Tutorials", pages: [ {title: "Queues on Railway", url: "https://blog.railway.app/p/queues"}, {title: "Gitlab CI/CD with Railway", url: "https://blog.railway.app/p/gitlab-ci-cd"}, + makePage("Set Up a Datadog Agent", "tutorials"), ] }, { diff --git a/src/docs/tutorials/set-up-a-datadog-agent.md b/src/docs/tutorials/set-up-a-datadog-agent.md new file mode 100644 index 000000000..c2b8124bc --- /dev/null +++ b/src/docs/tutorials/set-up-a-datadog-agent.md @@ -0,0 +1,187 @@ +--- +title: Set Up a Datadog Agent in Railway +--- + +Datadog provides a centralized location for logs, metrics, and traces emitted from applications deployed in various locations. + +Follow this tutorial to learn how to successfully deploy a Datadog agent in Railway and configure an application to send metrics and logs to it. + +**Prerequisites** +- Railway CLI installed +- Datadog API key and site value + +## Create the Project Structure + +First, from your local machine, create a folder for your project called `railway-project`. Inside of this folder, create two folders called `agent` and `expressapi`. + +``` +railway-project/ +├── agent/ +└── expressapi/ +``` + +Within the `agent` folder, we'll create the files necessary for the Datadog agent. In the `expressapi` folder we will create the application to send data to the agent which will be forwarded to Datadog. + +## Set Up the Datadog Agent + +Inside of the `agent` folder you just created, create three files - +- `Dockerfile` +- `syslog.yaml` +- `datadog.yaml` + +#### Define the Dockerfile + +The Dockerfile should be defined as follows - + +```dockerfile +FROM datadog/agent:7 + +# Set environment variables +ENV DD_LOGS_ENABLED=true +ENV DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL=true +ENV DD_DOGSTATSD_NON_LOCAL_TRAFFIC=true +ENV DD_BIND_HOST=::1 + +ARG DD_API_KEY +ARG DD_HOSTNAME +ARG DD_SITE + +# Copy your custom datadog.yaml into the container +COPY datadog.yaml /etc/datadog-agent/datadog.yaml + +# Copy the syslog configuration file +COPY syslog.yaml /etc/datadog-agent/conf.d/syslog.d/ + +# Expose the DogStatsD port and the syslog port +EXPOSE 8125/udp +EXPOSE 514/udp +``` + +#### Define the syslog.yaml file + +The `syslog.yaml` file is used to instruct the agent to listen for syslogs to be forwarded on the configured port. + +``` +logs: + - type: udp + port: 514 + service: "node-app" + source: syslog +``` + +#### Define the datadog.yaml file + +The `datadog.yaml` file is used to instruct the agent to send logs to Datadog over `http` instead of the default `tcp`. + +``` +logs_config: + force_use_http: true +``` + +## Set Up the Node Express App + +Inside of the `expressapi` folder you created, create an `app.js` file and use npm (or your preferred package manager) to install the required dependencies - + +```npm +npm i express winston winston-syslog +``` + +#### Define the app.js file + +The `app.js` file defines your express server. This is where we will initialize the StatsD client and the Winston logger, which will send metrics and logs, respectively, to the Datadog agent. + +```javascript +const express = require('express'); +const app = express(); + +const StatsD = require('hot-shots'); +const { createLogger, format, transports } = require('winston'); +require('winston-syslog').Syslog; +const port = process.env.PORT || 3000; + +// Configure the StatsD client +const statsdClient = new StatsD({ + host: process.env.DD_AGENT_HOST, + port: process.env.DD_AGENT_STATSD_PORT, + protocol: 'udp', + cacheDns: true, + udpSocketOptions: { + type: 'udp6', + reuseAddr: true, + ipv6Only: true, + }, +}); + +// Configure Winston logger +const logger = createLogger({ + level: 'info', + exitOnError: false, + format: format.json(), + transports: [ + new transports.Syslog({ + host: process.env.DD_AGENT_HOST, + port: process.env.DD_AGENT_SYSLOG_PORT, + protocol: 'udp6', + format: format.json(), + app_name: 'node-app', + }), + ], +}); + +app.get('/', (req, res) => { + // Increment a counter for the root path + statsdClient.increment('data_dog_example.homepage.hits'); + statsdClient.gauge('data_dog_example.homepage.hits', 124); + + // forward logs from root path + logger.info('Root route was accessed'); + + res.send('Hello World!'); +}); + +app.get('/test', (req, res) => { + // Increment a counter for the test path + statsdClient.increment('data_dog_example.testpage.hits'); + + // forward logs from test path + logger.info('Test route was accessed'); + + res.send('This is the test endpoint!'); +}); + +app.listen(port, () => { + console.log(`Example app listening at port ${port}`); +}); + +``` + +#### Winston and hot-shots + +In this example app, we are using `Winston` as the logger and `hot-shots` as the statsD client. + +- `Winston` is configured using `winston-syslog` to transport logs to the Datadog agent via Syslog over `udp6`. +- `hot-shots` is configured to send metrics to the Datadog agent over `udp6`. + +## Deploy to Railway + +Now that our project is defined, we can deploy the services to Railway. + +#### Create a Project and Services + +1. Using the Railway CLI, run the following command to create a new project - + + ``` + railway init + ``` + + Give you project a name when prompted. + +2. Open your project by running the following - + ```plaintext + railway open + ``` +3. Create a new Empty Service by clicking `+ New` and choosing `Empty Service`. Right click on the service, select `Update Info` and name it `datadog-agent`. + +4. Repeat step 3, but name this service `expressapi`. + +#### Use Railway Up \ No newline at end of file From e850cc2c3f393b8f84fc3d06126b210e0fe9a3e5 Mon Sep 17 00:00:00 2001 From: Melissa Hale Date: Fri, 15 Dec 2023 17:56:45 -0600 Subject: [PATCH 2/2] finish datadog guide --- src/docs/tutorials/migrate-from-heroku.md | 8 +- src/docs/tutorials/set-up-a-datadog-agent.md | 364 ++++++++++++------- 2 files changed, 245 insertions(+), 127 deletions(-) diff --git a/src/docs/tutorials/migrate-from-heroku.md b/src/docs/tutorials/migrate-from-heroku.md index 72a9b8b63..98539c35e 100644 --- a/src/docs/tutorials/migrate-from-heroku.md +++ b/src/docs/tutorials/migrate-from-heroku.md @@ -19,7 +19,7 @@ In your terminal, ensure your current working directory is the same directory wh This is important so that as you complete the following steps, the [Railway CLI](/reference/cli-api) is properly linked. -## Login to Railway from the CLI +## 1. Login to Railway from the CLI Ensure your CLI is authenticated to your Railway account: ```bash @@ -28,7 +28,7 @@ railway login This command will prompt to open a browser to complete authentication. Once authenticated, commands executed by the Railway CLI, will be performed in the context of your Railway account. -## Create a New Project +## 2. Create a New Project Now, let's create a new [project](/reference/projects): ```bash @@ -38,7 +38,7 @@ railway init This command will prompt you to define a name for your service. -## Deploy the Service +## 3. Deploy the Service Once your project is created, you can push your code into the project and assign a domain. @@ -60,7 +60,7 @@ railway domain Now the service will be available over the Internet via the provided domain. -## Migrate the Environment Variables +## 4. Migrate the Environment Variables Finally, we will import the environment variables from Heroku into Railway. diff --git a/src/docs/tutorials/set-up-a-datadog-agent.md b/src/docs/tutorials/set-up-a-datadog-agent.md index c2b8124bc..e7b5dd369 100644 --- a/src/docs/tutorials/set-up-a-datadog-agent.md +++ b/src/docs/tutorials/set-up-a-datadog-agent.md @@ -4,15 +4,37 @@ title: Set Up a Datadog Agent in Railway Datadog provides a centralized location for logs, metrics, and traces emitted from applications deployed in various locations. -Follow this tutorial to learn how to successfully deploy a Datadog agent in Railway and configure an application to send metrics and logs to it. +While Railway has a native, [centralized logging mechanism](/guides/logs#log-explorer), you may have a need to ship this data to another location, to view it alongside data collected from systems outside of Railway. + +**Objectives** + +In this tutorial you will learn how to - +- Deploy a Datadog agent in Railway, listening for metrics and logs. +- Configure an application to send metrics and logs to the agent. + +If you are looking for a quicker way to get started, you can also deploy this project from a template. **Prerequisites** -- Railway CLI installed + +To be successfull, you should already have - + +- Railway [CLI installed](guides/cli#installing-the-cli) - Datadog API key and site value -## Create the Project Structure +**Caveats** + +Keep in mind that the Datadog agent sends data to Datadog over the Internet, meaning you may see a spike in egress cost. + +## 1. Create the Project Structure + +First we'll create the project structure. -First, from your local machine, create a folder for your project called `railway-project`. Inside of this folder, create two folders called `agent` and `expressapi`. +From your local machine - + +- Create a folder for your project called `railway-project`. +- Create two folders inside of `railway-project` called `agent` and `expressapi`. + +You project structure should look like this ``` railway-project/ @@ -20,168 +42,264 @@ railway-project/ └── expressapi/ ``` -Within the `agent` folder, we'll create the files necessary for the Datadog agent. In the `expressapi` folder we will create the application to send data to the agent which will be forwarded to Datadog. -## Set Up the Datadog Agent +## 2. Set Up the Datadog Agent -Inside of the `agent` folder you just created, create three files - -- `Dockerfile` -- `syslog.yaml` -- `datadog.yaml` +Now we'll add files to the `agent` folder, which will build the Datadog Agent image. + +- Inside of the `agent` folder, create three files - + - `Dockerfile` + - `syslog.yaml` + - `datadog.yaml` #### Define the Dockerfile -The Dockerfile should be defined as follows - +Let's define the Dockerfile. -```dockerfile -FROM datadog/agent:7 +- Within your Dockerfile, add the following contents. -# Set environment variables -ENV DD_LOGS_ENABLED=true -ENV DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL=true -ENV DD_DOGSTATSD_NON_LOCAL_TRAFFIC=true -ENV DD_BIND_HOST=::1 + ```dockerfile + FROM datadog/agent:7 -ARG DD_API_KEY -ARG DD_HOSTNAME -ARG DD_SITE + # Set environment variables + ENV DD_LOGS_ENABLED=true + ENV DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL=true + ENV DD_DOGSTATSD_NON_LOCAL_TRAFFIC=true + ENV DD_BIND_HOST=::1 -# Copy your custom datadog.yaml into the container -COPY datadog.yaml /etc/datadog-agent/datadog.yaml + ARG DD_API_KEY + ARG DD_HOSTNAME + ARG DD_SITE -# Copy the syslog configuration file -COPY syslog.yaml /etc/datadog-agent/conf.d/syslog.d/ + # Copy the datadog.yaml into the container + COPY datadog.yaml /etc/datadog-agent/datadog.yaml -# Expose the DogStatsD port and the syslog port -EXPOSE 8125/udp -EXPOSE 514/udp -``` + # Copy the syslog configuration file into the container + COPY syslog.yaml /etc/datadog-agent/conf.d/syslog.d/ + + # Expose the StatsD port and the syslog port + EXPOSE 8125/udp + EXPOSE 514/udp + ``` #### Define the syslog.yaml file The `syslog.yaml` file is used to instruct the agent to listen for syslogs to be forwarded on the configured port. -``` -logs: - - type: udp - port: 514 - service: "node-app" - source: syslog -``` +- Within the `syslog.yaml` file, add the following contents - + + ``` + logs: + - type: udp + port: 514 + service: "node-app" + source: syslog + ``` #### Define the datadog.yaml file The `datadog.yaml` file is used to instruct the agent to send logs to Datadog over `http` instead of the default `tcp`. -``` -logs_config: - force_use_http: true -``` +- Within the `datadog.yaml` file, add the following contents - -## Set Up the Node Express App + ``` + logs_config: + force_use_http: true + ``` -Inside of the `expressapi` folder you created, create an `app.js` file and use npm (or your preferred package manager) to install the required dependencies - +## 3. Set Up the Node Express App -```npm -npm i express winston winston-syslog -``` +Now let's build a Node Express App that will send logs and metrics to the Datadog Agent over the [Private Network](/reference/private-networking). + +- Create an `app.js` file inside of the `expressapi` folder you created in Step 1. +- Use `npm` (or your preferred package manager) to install the required dependencies - + + ```npm + npm i express winston winston-syslog + ``` #### Define the app.js file The `app.js` file defines your express server. This is where we will initialize the StatsD client and the Winston logger, which will send metrics and logs, respectively, to the Datadog agent. -```javascript -const express = require('express'); -const app = express(); - -const StatsD = require('hot-shots'); -const { createLogger, format, transports } = require('winston'); -require('winston-syslog').Syslog; -const port = process.env.PORT || 3000; - -// Configure the StatsD client -const statsdClient = new StatsD({ - host: process.env.DD_AGENT_HOST, - port: process.env.DD_AGENT_STATSD_PORT, - protocol: 'udp', - cacheDns: true, - udpSocketOptions: { - type: 'udp6', - reuseAddr: true, - ipv6Only: true, - }, -}); - -// Configure Winston logger -const logger = createLogger({ - level: 'info', - exitOnError: false, - format: format.json(), - transports: [ - new transports.Syslog({ - host: process.env.DD_AGENT_HOST, - port: process.env.DD_AGENT_SYSLOG_PORT, - protocol: 'udp6', - format: format.json(), - app_name: 'node-app', - }), - ], -}); - -app.get('/', (req, res) => { - // Increment a counter for the root path - statsdClient.increment('data_dog_example.homepage.hits'); - statsdClient.gauge('data_dog_example.homepage.hits', 124); - - // forward logs from root path - logger.info('Root route was accessed'); - - res.send('Hello World!'); -}); - -app.get('/test', (req, res) => { - // Increment a counter for the test path - statsdClient.increment('data_dog_example.testpage.hits'); - - // forward logs from test path - logger.info('Test route was accessed'); - - res.send('This is the test endpoint!'); -}); - -app.listen(port, () => { - console.log(`Example app listening at port ${port}`); -}); - -``` +- Within the `app.js` file, add the following contents - + + ```javascript + const express = require('express'); + const app = express(); + + const StatsD = require('hot-shots'); + const { createLogger, format, transports } = require('winston'); + require('winston-syslog').Syslog; + const port = process.env.PORT || 3000; + + // Configure the StatsD client + const statsdClient = new StatsD({ + host: process.env.DD_AGENT_HOST, + port: process.env.DD_AGENT_STATSD_PORT, + protocol: 'udp', + cacheDns: true, + udpSocketOptions: { + type: 'udp6', + reuseAddr: true, + ipv6Only: true, + }, + }); + + // Configure Winston logger + const logger = createLogger({ + level: 'info', + exitOnError: false, + format: format.json(), + transports: [ + new transports.Syslog({ + host: process.env.DD_AGENT_HOST, + port: process.env.DD_AGENT_SYSLOG_PORT, + protocol: 'udp6', + format: format.json(), + app_name: 'node-app', + }), + ], + }); + + app.get('/', (req, res) => { + // Increment a counter for the root path + statsdClient.increment('data_dog_example.homepage.hits'); + statsdClient.gauge('data_dog_example.homepage.hits', 124); + + // forward logs from root path + logger.info('Root route was accessed'); + + res.send('Hello World!'); + }); + + app.get('/test', (req, res) => { + // Increment a counter for the test path + statsdClient.increment('data_dog_example.testpage.hits'); + + // forward logs from test path + logger.info('Test route was accessed'); + + res.send('This is the test endpoint!'); + }); + + app.listen(port, () => { + console.log(`Example app listening at port ${port}`); + }); + + ``` #### Winston and hot-shots -In this example app, we are using `Winston` as the logger and `hot-shots` as the statsD client. +In this example app, we are using `Winston` as the logger and `hot-shots` as the StatsD client. -- `Winston` is configured using `winston-syslog` to transport logs to the Datadog agent via Syslog over `udp6`. -- `hot-shots` is configured to send metrics to the Datadog agent over `udp6`. +- `Winston` is configured using `winston-syslog` to transport **logs** to the Datadog agent via Syslog over `udp6`. +- `hot-shots` is configured to send **metrics** to the Datadog agent over `udp6`. -## Deploy to Railway +## 4. Set Up the Railway Project -Now that our project is defined, we can deploy the services to Railway. +Now let's create the project using the CLI, then create the services and variables from within the project in Railway. -#### Create a Project and Services +You will need your **Datadog API key** and **Site** value in this step. -1. Using the Railway CLI, run the following command to create a new project - +If you have not already done so, please [install the CLI](/guides/cli#installing-the-cli) and [authenticate](/guides/cli#authenticating-with-the-cli). - ``` +#### Create a Project + +- In your terminal, run the following command to create a new project - + + ```plaintext railway init ``` - Give you project a name when prompted. +- Name your project `datadog-project` when prompted (you can change this later). + +- Open your project in Railway by running the following - -2. Open your project by running the following - ```plaintext railway open ``` -3. Create a new Empty Service by clicking `+ New` and choosing `Empty Service`. Right click on the service, select `Update Info` and name it `datadog-agent`. - -4. Repeat step 3, but name this service `expressapi`. -#### Use Railway Up \ No newline at end of file +#### Create the Services + +- In Railway, create an Empty Service by clicking `+ New` button in the top right-hand corner and choosing `Empty Service` in the prompt. +- Right click on the service that is created, select `Update Info` and name it `datadog-agent`. +- Repeat the above steps to create a second service, but name the second service `expressapi`. + +#### Add the Variables + +Each service requires unique variables listed below. For each service, follow the steps to add the variables required for the service. + +Variables - +
+
+ ```dockerfile + DD_API_KEY= + DD_HOSTNAME=${{RAILWAY_PRIVATE_DOMAIN}} + DD_SITE= + ``` +

`datadog-agent` variables

+
+
+ ```dockerfile + DD_AGENT_HOST=${{datadog-agent.DD_HOSTNAME}} + DD_AGENT_STATSD_PORT=8125 + DD_AGENT_SYSLOG_PORT=514 + ``` +

`expressapi` variables

+
+
+ +- Click on the service card +- Click on the `Variables` tab +- Click on `Raw Editor` +- Paste the required variables (be sure to update the Datadog API key and site with your own values) +- Click `Update Variables` and `Deploy` + +## 5. Deploy to Railway + +Now we're ready to deploy our services. We will use the CLI to push code from our local machine to Railway. + +#### Railway Up + +Follow these steps for each service - + +- In your local terminal, change directory into the `agent` folder. +- Link to `datadog-project` by running the following command - + ```plaintext + railway link + ``` +- Follow the prompts, selecting the `datadog-project` and `production` environment. +- Link to the `datadog-agent` service by running the following command - + ```plaintext + railway service + ``` +- Follow the prompt, selecting the `datadog-agent` service. +- Deploy the agent by running the following command - + ```plaintext + railway up -d + ``` +- Change directory into your `expressapi` folder and repeat the steps above, but for the `expressapi` service. + +#### Create a Domain for the Express App + +The express app will send logs and metrics to the Datadog agent upon navigation to either of its two routes. So let's give it a domain - +- Ensure that you are linked to the `datadog-project` and `expressapi` service (refer to the steps above) +- Assign the `expressapi` a domain by running the following command - + ```plaintext + railway domain + ``` + +## 6. Test and Confirm + +Test that your Datadog Agent is receiving and forwarding data to Datadog by navigating to the routes in the Express app - +- `/` +- `/test` + +Generate some traffic to these two routes and verify in your Datadog instance that the data is there. + +## Conclusion + +Congratulations! You have deployed a Datadog Agent and a Node Express app that sends logs and metrics to Datadog. + +This is a *very* basic implementation, and you should refer to the Datadog documentation for information on how to customize the data you send. \ No newline at end of file