diff --git a/packages/generators/__tests__/add-generator.test.ts b/packages/generators/__tests__/add-generator.test.ts
deleted file mode 100644
index 85b17d76c8d..00000000000
--- a/packages/generators/__tests__/add-generator.test.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-import { generatePluginName } from "../utils";
-
-describe("generatePluginName", () => {
-	it("should return webpack Standard Plugin Name for Name : extract-text-webpack-plugin", () => {
-		const pluginName = generatePluginName("extract-text-webpack-plugin");
-		expect(pluginName).toEqual("ExtractTextWebpackPlugin");
-	});
-
-	it("should return webpack Standard Plugin Name for Name : webpack.DefinePlugin", () => {
-		const pluginName = generatePluginName("webpack.DefinePlugin");
-		expect(pluginName).toEqual("webpack.DefinePlugin");
-	});
-});
diff --git a/packages/generators/__tests__/plugin-generator.test.ts b/packages/generators/__tests__/plugin-generator.test.ts
new file mode 100644
index 00000000000..51d87400010
--- /dev/null
+++ b/packages/generators/__tests__/plugin-generator.test.ts
@@ -0,0 +1,44 @@
+import { join } from 'path';
+import { run } from 'yeoman-test';
+const assert = require('yeoman-assert');
+import { generatePluginName } from '../utils';
+
+describe('plugin generator', () => {
+    it('generates a default plugin', async () => {
+        const outputDir = await run(join(__dirname, '../plugin-generator'));
+        const pluginDir = `${outputDir}/my-webpack-plugin`;
+        const srcFiles = ['cjs.js', 'index.js'];
+        const testFiles = ['functional.test.js', 'test-utils.js'];
+        const exampleFiles = ['webpack.config.js', 'src/index.js', 'src/lazy-module.js', 'src/static-esm-module.js'];
+
+        // Check that files in all folders are scaffolded. Checking them separately so we know which directory has the problem
+        // assert for src files
+        assert.file([...srcFiles.map(file => `${pluginDir}/src/${file}`)]);
+
+        // assert for test files
+        assert.file([...testFiles.map(file => `${pluginDir}/test/${file}`)]);
+
+        // assert for example files
+        assert.file([...exampleFiles.map(file => `${pluginDir}/examples/simple/${file}`)]);
+
+        // Check the contents of the webpack config and loader file
+        assert.fileContent([
+            [`${pluginDir}/examples/simple/webpack.config.js`, 'new MyWebpackPlugin()'],
+            [`${pluginDir}/src/index.js`, 'MyWebpackPlugin.prototype.apply = function(compiler) {'],
+        ]);
+
+        // higher timeout so travis has enough time to execute
+    }, 10000);
+});
+
+describe('generate plugin name', () => {
+    it('should return webpack Standard Plugin Name for Name : extract-text-webpack-plugin', () => {
+        const pluginName = generatePluginName('extract-text-webpack-plugin');
+        expect(pluginName).toEqual('ExtractTextWebpackPlugin');
+    });
+
+    it('should return webpack Standard Plugin Name for Name : webpack.DefinePlugin', () => {
+        const pluginName = generatePluginName('webpack.DefinePlugin');
+        expect(pluginName).toEqual('webpack.DefinePlugin');
+    });
+});