Access Parameter store config locally via Dotenv

dotcom-rendering/docs/values/README.md

@@ -0,0 +1,15 @@
+# Values
+
+## Approachable
+
+The project should have as few barriers as possible for anyone to work on it, including designers, UX, people outside of the Guardian and more.
+
+### Lines in the sand
+
+#### `.env` shouldn't be required
+
+While we use a `.env` file for configuration and secrets for `dotcom-rendering` to function correctly in `PROD`, not having it shouldn't block someone from running the project altogether.
+
+If you're considering adding something to the `.env` file - please consult with the dotcom team first. Wherever possible seeking solutions like passing the data from [Frontend](https://github.com/guardian/frontend) is preferred.
+
+e.g. [Fallback for images salt](#) (In development so not linked to)

dotcom-rendering/makefile

@@ -30,13 +30,15 @@ deploy:
 
 # prod #########################################
 
-build: clean-dist install
+build: export NODE_ENV=production
+build: clean-dist install gen-dotenv
 	$(call log, "building production bundles")
-	@NODE_ENV=production webpack --config ./scripts/webpack/webpack.config.js --progress
+	@webpack --config ./scripts/webpack/webpack.config.js --progress
 
-build-ci: clean-dist install
+build-ci: export NODE_ENV=production
+build-ci: clean-dist install gen-dotenv
 	$(call log, "building production bundles")
-	@NODE_ENV=production SKIP_LEGACY=true webpack --config ./scripts/webpack/webpack.config.js
+	@SKIP_LEGACY=true webpack --config ./scripts/webpack/webpack.config.js
 
 start-ci: install
 	$(call log, "starting PROD server...")
@@ -66,11 +68,11 @@ run-ci: stop build-ci start-ci
 
 # dev #########################################
 
-dev: clear clean-dist install
+dev: clear clean-dist install gen-dotenv
 	$(call log, "starting DEV server")
 	@NODE_ENV=development SKIP_LEGACY=true webpack serve --config ./scripts/webpack/webpack.config.js
 
-dev-legacy: clear clean-dist install
+dev-legacy: clear clean-dist install gen-dotenv
 	$(call log, "starting DEV server")
 	@NODE_ENV=development webpack serve --config ./scripts/webpack/webpack.config.js
 
@@ -177,6 +179,10 @@ gen-fixtures:
 	$(call log, "Generating new article fixture data")
 	@node scripts/test-data/gen-fixtures.js
 
+gen-dotenv:
+	$(call log, "Generating .env file")
+	@node scripts/dotenv/gen-dotenv.js
+
 perf-test:
 	@node scripts/perf/perf-test.js
 

dotcom-rendering/package.json

@@ -128,6 +128,7 @@
     "cypress-wait-until": "^1.7.1",
     "cypress-webpack-preprocessor-v5": "^5.0.0-alpha.1",
     "doctoc": "^2.1.0",
+    "dotenv-webpack": "^7.1.0",
     "dompurify": "^2.3.6",
     "dynamic-import-polyfill": "^0.1.1",
     "eslint": "^7.24.0",

dotcom-rendering/scripts/dotenv/gen-dotenv.js

@@ -0,0 +1,87 @@
+const { CredentialsProviderError } = require('@aws-sdk/property-provider');
+const path = require('path');
+const fs = require('fs').promises;
+const { prompt, log, warn } = require('../env/log');
+const secrets = require('../secrets');
+const { getAwsSsmParameters } = require('./get-aws-ssm-parameters');
+
+const ENV_PATH = path.resolve(__dirname, '../../.env');
+
+const checkEnv = async () => {
+	try {
+		const env = (await fs.readFile(ENV_PATH)).toString();
+
+		let valid = true;
+		for (const secret of secrets) {
+			const regex = new RegExp(`^${secret.key.replace('.', '\\.')}=.*`);
+			if (!env.match(regex)) valid = false;
+		}
+
+		return valid;
+	} catch (_err) {
+		return false;
+	}
+};
+
+const genEnv = async () => {
+	const env = process.env.NODE_ENV === 'production' ? 'prod' : 'dev';
+	const parameters = await getAwsSsmParameters(env);
+
+	let envString = '';
+	for (const secret of secrets) {
+		envString += `${secret.key}=${parameters[secret.key]}\n`;
+	}
+
+	await fs.writeFile(ENV_PATH, envString);
+};
+
+// eslint-disable-next-line @typescript-eslint/no-floating-promises
+(async () => {
+	try {
+		const validEnv = await checkEnv();
+		if (!validEnv) {
+			log(
+				'[scripts/dotenv] .env file is missing, attemting to generate it from AWS parameters...',
+			);
+
+			await genEnv();
+
+			log('[scripts/dotenv] .env file written successfully');
+		} else {
+			log('[scripts/dotenv] valid .env file exists, moving on...');
+		}
+	} catch (err) {
+		if (err instanceof CredentialsProviderError) {
+			const PROD = process.env.NODE_ENV === 'production';
+			if (PROD) {
+				warn(
+					'[scripts/dotenv] could not generate .env file from AWS Parameter Store. Exiting',
+				);
+				throw err;
+			}
+
+			prompt(
+				'[scripts/dotenv] Could not load AWS credentials to generate .env file',
+				"[scripts/dotenv] This won't stop dotcom-rendering from working, it will just vary from PROD by:",
+			);
+
+			for (const secret of secrets) {
+				prompt(
+					`[scripts/dotenv]  * ${secret.key}: ${secret.missingMessage}`,
+				);
+			}
+
+			prompt(
+				'',
+				'[scripts/dotenv] To get things working PROD like either:',
+				'[scripts/dotenv]  * Get your credentials from Janus',
+				'[scripts/dotenv]  * Ask a local engineer for a copy of the .env file',
+				'[scripts/dotenv] Then try again.',
+			);
+
+			process.exit(0);
+		} else {
+			throw err;
+		}
+	}
+})();

dotcom-rendering/scripts/dotenv/get-aws-ssm-parameters.js

@@ -0,0 +1,56 @@
+const {
+	GetParametersByPathCommand,
+	SSMClient,
+} = require('@aws-sdk/client-ssm');
+
+process.env.AWS_PROFILE = 'frontend';
+const ssm = new SSMClient({ region: 'eu-west-1' });
+const env = process.env.NODE_ENV === 'production' ? 'prod' : 'dev';
+
+async function* scrollParameters(params) {
+	let command = new GetParametersByPathCommand(params);
+	let response = await ssm.send(command);
+
+	while (true) {
+		const parameters = response.Parameters;
+
+		// SSM returns undefined if there are no parameters
+		if (parameters === undefined) {
+			break;
+		}
+
+		for (const parameter of parameters) {
+			yield parameter;
+		}
+
+		if (!response.NextToken) {
+			break;
+		}
+
+		command = new GetParametersByPathCommand({
+			Path: `/dotcom/${env}/`,
+			Recursive: true,
+			WithDecryption: true,
+			NextToken: response.NextToken,
+		});
+		response = ssm.send(command);
+	}
+}
+
+async function getAwsSsmParameters() {
+	const parameters = {};
+	const params = {
+		Path: `/dotcom/${env}/`,
+		Recursive: true,
+		WithDecryption: true,
+	};
+
+	for await (const parameter of scrollParameters(params)) {
+		const key = parameter.Name.replace(`/dotcom/${env}/`, '');
+		parameters[key] = parameter.Value;
+	}
+
+	return parameters;
+}
+
+module.exports = { getAwsSsmParameters };

dotcom-rendering/scripts/secrets.js

@@ -0,0 +1,8 @@
+const secrets = [
+	{
+		key: 'IMAGE_SALT',
+		missingMessage: 'Images will fallback to a placeholder image',
+	},
+];
+
+module.exports = secrets;

dotcom-rendering/scripts/webpack/@types/dotenv-webpack/index.d.ts

@@ -0,0 +1 @@
+declare module 'dotenv-webpack';

dotcom-rendering/scripts/webpack/webpack.config.server.js

@@ -1,4 +1,5 @@
 // @ts-check
+const Dotenv = require('dotenv-webpack');
 const GuStatsReportPlugin = require('./plugins/gu-stats-report-plugin');
 
 const DEV = process.env.NODE_ENV === 'development';
@@ -60,6 +61,7 @@ module.exports = ({ sessionId }) => ({
 					team: 'dotcom',
 					sessionId,
 				}),
+				new Dotenv(),
 		  ]
 		: undefined,
 	module: {

dotcom-rendering/src/server/prod-server.ts

@@ -14,11 +14,6 @@ import {
 	render as renderAMPArticle,
 	renderPerfTest as renderAMPArticlePerfTest,
 } from '../amp/server';
-import { log, warn } from '../../scripts/env/log';
-import {
-	getGuardianConfiguration,
-	GuardianConfiguration,
-} from './lib/aws/aws-parameters';
 import { recordBaselineCloudWatchMetrics } from './lib/aws/metrics-baseline';
 import { logger } from './lib/logging';
 import { getContentFromURLMiddleware } from './lib/get-content-from-url';
@@ -39,17 +34,6 @@ const logRenderTime = responseTime(
 export const prodServer = () => {
 	logger.info('dotcom-rendering is GO.');
 
-	if (process.env.DISABLE_LOGGING_AND_METRICS !== 'true') {
-		getGuardianConfiguration('prod')
-			.then((config: GuardianConfiguration) => {
-				log(`loaded ${config.size()} configuration parameters`);
-			})
-			.catch((err: any) => {
-				warn('Failed to get configuration. Bad AWS credentials?');
-				warn(err);
-			});
-	}
-
 	const app = express();
 
 	app.use(express.json({ limit: '50mb' }));