Merge pull request #67 from f-lab-edu/feature/66-typeorm-seed-problem

[#66] typeorm의 seed 라이브러리의 문제점

data-source.ts

@@ -34,6 +34,14 @@ const options: DataSourceOptions & SeederOptions = {
     logging: true,
     seeds: [`src/database/seeds/**/*{.js,.ts}`],
     factories: [`src/database/factories/**/*{.js,.ts}`],
+    extra: {
+        connectionLimit: 30,
+        queueLimit: 0,
+        waitForConnections: true,
+    },
+    // 커넥션 풀 사이즈 설정
+    poolSize: 30,
+    connectTimeout: 120000,
 };
 
 export const dataSource = new DataSource(options);

docker-compose.yml

@@ -29,6 +29,7 @@ services:
       container_name: mysql_prod
       volumes:
           - ./mysql-data:/var/lib/mysql
+          - ./mysql/my.cnf:/etc/mysql/conf.d/my.cnf
       ports:
           - "${MYSQL_OUTBOUND_PORT}:${MYSQL_INBOUND_PORT}"
       environment:

mysql/my.cnf

@@ -0,0 +1,11 @@
+[mysqld]
+innodb_stats_auto_recalc = 1
+innodb_stats_persistent = 1
+character-set-server = utf8mb4
+collation-server = utf8mb4_unicode_ci
+
+[client]
+default-character-set = utf8mb4
+
+[mysql]
+default-character-set = utf8mb4

src/database/factories/article.factory.ts

@@ -2,18 +2,45 @@ import { setSeederFactory } from "typeorm-extension";
 
 import { ArticleEntity } from "@APP/entities/article.entity";
 
+import { MEMBER_ID_RANGE } from "../utils/member-id-range.const";
+
+// export default setSeederFactory(ArticleEntity, async (faker) => {
+//     const article = new ArticleEntity();
+//     const startDate = faker.date.recent({ days: 365 });
+
+//     article.title = faker.lorem.sentence();
+//     article.content = faker.lorem.paragraph();
+//     article.startTime = faker.date.recent({ days: 365 });
+//     article.endTime = faker.date.soon({ days: 1, refDate: article.startTime });
+//     article.articleImage = faker.image.url();
+//     article.memberId = faker.number.int({ min: 1, max: 100 });
+//     article.categoryId = faker.number.int({ min: 1, max: 16 });
+//     article.regionId = faker.number.int({ min: 1, max: 1 });
+//     article.districtId = faker.number.int({ min: 1, max: 129 });
+//     article.createdAt = startDate;
+//     article.updatedAt = startDate;
+
+//     return article;
+// });
+
 export default setSeederFactory(ArticleEntity, async (faker) => {
     const article = new ArticleEntity();
     const startDate = faker.date.recent({ days: 365 });
 
-    article.title = faker.lorem.sentence();
+    const memberIdRanges = faker.helpers.weightedArrayElement(MEMBER_ID_RANGE);
+
+    article.memberId = faker.number.int({
+        min: memberIdRanges.min,
+        max: memberIdRanges.max,
+    });
+
+    article.title = faker.lorem.sentence({ min: 5, max: 10 }).slice(0, 20);
     article.content = faker.lorem.paragraph();
     article.startTime = faker.date.recent({ days: 365 });
     article.endTime = faker.date.soon({ days: 1, refDate: article.startTime });
     article.articleImage = faker.image.url();
-    article.memberId = faker.number.int({ min: 1, max: 100 });
     article.categoryId = faker.number.int({ min: 1, max: 16 });
-    article.regionId = faker.number.int({ min: 1, max: 2 });
+    article.regionId = faker.number.int({ min: 1, max: 1 });
     article.districtId = faker.number.int({ min: 1, max: 129 });
     article.createdAt = startDate;
     article.updatedAt = startDate;

src/database/factories/articlelike.factory.ts

@@ -2,11 +2,34 @@ import { setSeederFactory } from "typeorm-extension";
 
 import { ArticleLikeEntity } from "@APP/entities/article-like.entity";
 
+import { ARTICLE_ID_RANGE } from "../utils/article-id-range.const";
+import { MEMBER_ID_RANGE } from "../utils/member-id-range.const";
+
+// export default setSeederFactory(ArticleLikeEntity, async () => {
+//     const articleLike = new ArticleLikeEntity();
+
+//     articleLike.articleId = 10000;
+//     articleLike.memberId = 10000;
+
+//     return articleLike;
+// });
+
 export default setSeederFactory(ArticleLikeEntity, async (faker) => {
     const articleLike = new ArticleLikeEntity();
 
-    articleLike.articleId = faker.number.int({ min: 1, max: 50000 });
-    articleLike.memberId = faker.number.int({ min: 1, max: 100 });
+    const memberIdRanges = faker.helpers.weightedArrayElement(MEMBER_ID_RANGE);
+
+    const articleRange = faker.helpers.weightedArrayElement(ARTICLE_ID_RANGE);
+
+    articleLike.memberId = faker.number.int({
+        min: memberIdRanges.min,
+        max: memberIdRanges.max,
+    });
+
+    articleLike.articleId = faker.number.int({
+        min: articleRange.min,
+        max: articleRange.max,
+    });
 
     return articleLike;
 });

src/database/factories/black-list.factory.ts

@@ -2,20 +2,22 @@ import { setSeederFactory } from "typeorm-extension";
 
 import { BlackListEntity } from "@APP/entities/black-list.entity";
 
+import { MEMBER_ID_RANGE } from "../utils/member-id-range.const";
+
 export default setSeederFactory(BlackListEntity, async (faker) => {
     const blackListEntry = new BlackListEntity();
 
-    let initiatorId = faker.number.int({ min: 1, max: 100 });
-    let targetId = faker.number.int({ min: 1, max: 100 });
+    const memberIdRanges = faker.helpers.weightedArrayElement(MEMBER_ID_RANGE);
 
-    // 자기 자신을 블랙 하는것을 방지 하기 위해
-    while (initiatorId === targetId) {
-        initiatorId = faker.number.int({ min: 1, max: 100 });
-        targetId = faker.number.int({ min: 1, max: 100 });
-    }
+    blackListEntry.blackerId = faker.number.int({
+        min: memberIdRanges.min,
+        max: memberIdRanges.max,
+    });
 
-    blackListEntry.blackerId = initiatorId;
-    blackListEntry.blackedId = targetId;
+    blackListEntry.blackedId = faker.number.int({
+        min: memberIdRanges.min,
+        max: memberIdRanges.max,
+    });
 
     return blackListEntry;
 });

src/database/factories/member.factory.ts

@@ -1,15 +1,20 @@
-import * as bcrypt from "bcrypt";
 import { setSeederFactory } from "typeorm-extension";
 
 import { MemberEntity } from "../../entities/member.entity";
 
 export default setSeederFactory(MemberEntity, async (faker) => {
-    const hashedPassword = await bcrypt.hash("123456", 10);
+    // bcrypt 로 해싱하면 너무 느려서 미리 해싱
+    const hashedPassword =
+        "$2b$10$2AzriinQGFUi0HE2LNReqeWopVZKYMKlAM9t0TjGRztoCdpSsnXva";
+
+    const timestamp = Date.now();
+    const randomString = Math.random().toString(36).substring(2, 8);
+    const email = `user_${timestamp}_${randomString}@${faker.internet.domainName()}`;
 
     const member = new MemberEntity();
     member.name = faker.person.firstName();
     member.nickname = faker.person.lastName();
-    member.email = faker.internet.email();
+    member.email = email;
     member.password = hashedPassword;
     member.isEmailVerified = true;
     member.profileImage = Math.random() > 0.5 ? faker.image.avatar() : null;

src/database/factories/participation.factory.ts

@@ -3,11 +3,26 @@ import { setSeederFactory } from "typeorm-extension";
 import { ParticipationStatus } from "@APP/common/enum/participation-status.enum";
 import { ParticipationEntity } from "@APP/entities/participation.entity";
 
+import { ARTICLE_ID_RANGE } from "../utils/article-id-range.const";
+import { MEMBER_ID_RANGE } from "../utils/member-id-range.const";
+
 export default setSeederFactory(ParticipationEntity, async (faker) => {
     const participation = new ParticipationEntity();
 
-    participation.articleId = faker.number.int({ min: 1, max: 50000 });
-    participation.memberId = faker.number.int({ min: 1, max: 100 });
+    const memberIdRanges = faker.helpers.weightedArrayElement(MEMBER_ID_RANGE);
+
+    const articleRange = faker.helpers.weightedArrayElement(ARTICLE_ID_RANGE);
+
+    participation.memberId = faker.number.int({
+        min: memberIdRanges.min,
+        max: memberIdRanges.max,
+    });
+
+    participation.articleId = faker.number.int({
+        min: articleRange.min,
+        max: articleRange.max,
+    });
+
     participation.status =
         Math.random() > 0.8
             ? ParticipationStatus.ACTIVE

src/database/factories/refresh-token.factory.ts

@@ -1,24 +1,13 @@
-import * as bcrypt from "bcrypt";
-import jwt from "jsonwebtoken";
 import { setSeederFactory } from "typeorm-extension";
 
-import { ENV_JWT_SECRET_KEY } from "@APP/common/constants/env-keys.const";
 import { RefreshTokenEntity } from "@APP/entities/refresh-token.entity";
 
-export default setSeederFactory(RefreshTokenEntity, async (faker) => {
-    const payload = {
-        email: faker.internet.email(),
-        sub: faker.number.int(),
-        type: "refresh",
-    };
-
-    const secret = process.env[ENV_JWT_SECRET_KEY] || "secret";
-    const expiresIn = "3600";
-
+export default setSeederFactory(RefreshTokenEntity, async () => {
     const refreshToken = new RefreshTokenEntity();
 
-    const token = jwt.sign(payload, secret, { expiresIn });
-    refreshToken.token = await bcrypt.hash(token, 10);
+    // bcrypt 해시가 너무 오래 걸려 미리 해싱
+    refreshToken.token =
+        "$2b$10$ndYzoPCCXt.24qzOusAB/OYMqaS1UKVqQsGvQh9t/MaqFqgg.g0fO";
 
     return refreshToken;
 });

src/database/seeds/article.seeder.ts

@@ -3,15 +3,77 @@ import { Seeder, SeederFactoryManager } from "typeorm-extension";
 
 import { ArticleEntity } from "@APP/entities/article.entity";
 
+// export default class ArticleSeeder implements Seeder {
+//     public async run(
+//         _dataSource: DataSource,
+//         factoryManager: SeederFactoryManager,
+//     ): Promise<void> {
+//         const articleFactory = factoryManager.get(ArticleEntity);
+
+//         const batchSize = 30_00;
+//         const totalArticles = 300_00;
+//         const iterations = totalArticles / batchSize;
+
+//         let totalCreated = 0;
+
+//         for (let i = 0; i < iterations; i++) {
+//             const articles = await articleFactory.saveMany(batchSize);
+//             totalCreated += articles.length;
+//             console.log(
+//                 `${i + 1}번째 배치 게시글 생성 완료: ${articles.length}`,
+//             );
+//         }
+
+//         console.log(`총 게시글 생성 완료: ${totalCreated}`);
+//     }
+// }
+
 export default class ArticleSeeder implements Seeder {
     public async run(
-        _dataSource: DataSource,
+        dataSource: DataSource,
         factoryManager: SeederFactoryManager,
     ): Promise<void> {
         const articleFactory = factoryManager.get(ArticleEntity);
+        const queryRunner = dataSource.createQueryRunner();
+
+        const batchSize = 1000;
+        const totalArticles = 100000;
+        const iterations = Math.ceil(totalArticles / batchSize);
+
+        let totalCreated = 0;
+
+        try {
+            await queryRunner.connect();
+            await queryRunner.startTransaction();
+
+            for (let i = 0; i < iterations; i++) {
+                const articles = await Promise.all(
+                    Array.from({ length: batchSize }, () =>
+                        articleFactory.make(),
+                    ),
+                );
+
+                const result = await queryRunner.manager
+                    .createQueryBuilder()
+                    .insert()
+                    .into(ArticleEntity)
+                    .values(articles)
+                    .orIgnore()
+                    .execute();
 
-        const articles = await articleFactory.saveMany(1);
+                totalCreated += result.identifiers.length;
+                console.log(
+                    `${i + 1}번째 배치 게시글 생성 완료: ${result.identifiers.length}`,
+                );
+            }
 
-        console.log(`게시글 생성 완료 : ${articles.length}`);
+            await queryRunner.commitTransaction();
+            console.log(`총 게시글 생성 완료: ${totalCreated}`);
+        } catch (error) {
+            await queryRunner.rollbackTransaction();
+            throw error;
+        } finally {
+            await queryRunner.release();
+        }
     }
 }

src/database/seeds/articlelike.seeder.ts

@@ -3,15 +3,65 @@ import { Seeder, SeederFactoryManager } from "typeorm-extension";
 
 import { ArticleLikeEntity } from "@APP/entities/article-like.entity";
 
+// export default class ArticleLikeSeeder implements Seeder {
+//     public async run(
+//         _dataSource: DataSource,
+//         factoryManager: SeederFactoryManager,
+//     ): Promise<void> {
+//         const articleLikeFactory = factoryManager.get(ArticleLikeEntity);
+
+//         await articleLikeFactory.saveMany(180000);
+
+//         console.log(`게시글 좋아요 생성 완료`);
+//     }
+// }
+
 export default class ArticleLikeSeeder implements Seeder {
     public async run(
-        _dataSource: DataSource,
+        dataSource: DataSource,
         factoryManager: SeederFactoryManager,
     ): Promise<void> {
         const articleLikeFactory = factoryManager.get(ArticleLikeEntity);
+        const queryRunner = dataSource.createQueryRunner();
+
+        const batchSize = 1000;
+        const totalLikes = 100000;
+        const iterations = Math.ceil(totalLikes / batchSize);
+
+        let totalCreated = 0;
+
+        try {
+            await queryRunner.connect();
+            await queryRunner.startTransaction();
+
+            for (let i = 0; i < iterations; i++) {
+                const articleLikes = await Promise.all(
+                    Array.from({ length: batchSize }, () =>
+                        articleLikeFactory.make(),
+                    ),
+                );
+
+                const result = await queryRunner.manager
+                    .createQueryBuilder()
+                    .insert()
+                    .into(ArticleLikeEntity)
+                    .values(articleLikes)
+                    .orIgnore()
+                    .execute();
 
-        await articleLikeFactory.saveMany(180000);
+                totalCreated += result.identifiers.length;
+                console.log(
+                    `${i + 1}번째 배치 게시글 좋아요 생성 완료: ${result.identifiers.length}`,
+                );
+            }
 
-        console.log(`게시글 좋아요 생성 완료`);
+            await queryRunner.commitTransaction();
+            console.log(`총 게시글 좋아요 생성 완료: ${totalCreated}`);
+        } catch (error) {
+            await queryRunner.rollbackTransaction();
+            throw error;
+        } finally {
+            await queryRunner.release();
+        }
     }
 }