-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathprepare_dataset.py
101 lines (79 loc) · 2.81 KB
/
prepare_dataset.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
import cv2
import audioread
import logging
import gc
import os
import sys
sys.path.append('../input/pytorch-image-models/pytorch-image-models-master')
import random
import time
import warnings
import glob
import librosa
import numpy as np
import pandas as pd
import soundfile as sf
import timm
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data as torchdata
from contextlib import contextmanager
from joblib import Parallel, delayed
from pathlib import Path
from typing import Optional
from sklearn.model_selection import StratifiedKFold, GroupKFold
from albumentations.core.transforms_interface import ImageOnlyTransform
from torchlibrosa.stft import LogmelFilterBank, Spectrogram
from torchlibrosa.augmentation import SpecAugmentation
from tqdm import tqdm
import albumentations as A
import albumentations.pytorch.transforms as T
import matplotlib.pyplot as plt
SR = 32000
USE_SEC = 30 # 90 # 60
TARGET_PATH = 'dataset/train_np_2022/'
def Audio_to_Array(path):
y, sr = sf.read(path, always_2d=True)
y = np.mean(y, 1) # there is (X, 2) array
if len(y) > SR:
y = y[SR:-SR]
if len(y) > SR * USE_SEC:
y = y[:SR * USE_SEC]
return y
def save_(path):
# save_path = "dataset/train_np/" + "/".join(path.split('/')[-2:])
save_path = TARGET_PATH + "/".join(path.split('/')[-2:])
np.save(save_path, Audio_to_Array(path))
AUDIO_PATH = 'dataset/birdclef-2022/train_audio'
train = pd.read_csv('dataset/birdclef-2022/train_metadata.csv')
train["file_path"] = AUDIO_PATH + '/' + train['filename']
CLASSES = sorted(os.listdir(AUDIO_PATH))
NUM_WORKERS = 23
paths = train["file_path"].values
for dir_ in tqdm(CLASSES):
_ = os.makedirs(TARGET_PATH + dir_, exist_ok=True)
_ = Parallel(n_jobs=NUM_WORKERS)(delayed(save_)(AUDIO_PATH) for AUDIO_PATH in tqdm(paths))
########## 2021
# TARGET_PATH = 'dataset/train_np_2021/'
# def Audio_to_Array(path):
# y, sr = sf.read(path, always_2d=True)
# y = np.mean(y, 1) # there is (X, 2) array
# if len(y) > SR:
# y = y[SR:-SR]
# if len(y) > SR * USE_SEC:
# y = y[:SR * USE_SEC]
# return y
# def save_(path):
# # save_path = "dataset/train_np/" + "/".join(path.split('/')[-2:])
# save_path = TARGET_PATH + "/".join(path.split('/')[-2:])
# np.save(save_path, Audio_to_Array(path))
# AUDIO_PATH = 'dataset/birdclef-2021/train_short_audio/'
# train = pd.read_csv('dataset/birdclef-2021/train_metadata.csv')
# train["file_path"] = 'dataset/birdclef-2021/train_short_audio/' + train['primary_label'] + '/' + train['filename']
# CLASSES = sorted(os.listdir(AUDIO_PATH))
# NUM_WORKERS = 23
# paths = train["file_path"].values
# for dir_ in tqdm(CLASSES):
# _ = os.makedirs(TARGET_PATH + dir_, exist_ok=True)
# _ = Parallel(n_jobs=NUM_WORKERS)(delayed(save_)(AUDIO_PATH) for AUDIO_PATH in tqdm(paths))