accuracy was almost zero

[zgsDream]

Hello, @qianjinhao can you provide an example of testing validation on mnist?Because when I used mnist verification, the accuracy was almost zero, but I felt that there was no problem with your implementation of loss, which bothered me very much. The following is my test code.thanks

import torch
from torch import nn, Tensor
from torch.optim import SGD
from torch.utils.data import DataLoader
from torchvision.datasets import MNIST
from torchvision.transforms import ToTensor
from tqdm import tqdm

from CircleLoss import CircleLoss

class NormLinear(nn.Linear):
    def __init__(self, in_features: int, out_features: int) -> None:
        super(NormLinear, self).__init__(in_features, out_features, bias=False)

    def forward(self, inp: Tensor) -> Tensor:
        return nn.functional.linear(nn.functional.normalize(inp),
                                    nn.functional.normalize(self.weight))

def get_loader(is_train: bool, batch_size: int) -> DataLoader:
    return DataLoader(
        dataset=MNIST(root="./mnist_data", train=is_train, transform=ToTensor(), download=True),
        batch_size=batch_size,
        shuffle=is_train,
    )


class Model(nn.Module):
    def __init__(self) -> None:
        super(Model, self).__init__()
        self.feature_extractor = nn.Sequential(
            nn.Conv2d(in_channels=1, out_channels=8, kernel_size=5),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2),
            nn.Conv2d(in_channels=8, out_channels=16, kernel_size=5),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2),
            nn.Conv2d(in_channels=16, out_channels=32, kernel_size=3),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2),
        )
        self.linear = NormLinear(32, 10)

    def forward(self, inp: Tensor) -> Tensor:
        feature = self.feature_extractor(inp).mean(dim=[2, 3])
        return self.linear(feature)


def main():
    model = Model()
    optimizer = SGD(model.parameters(), lr=0.01, momentum=0.9)
    train_loader = get_loader(is_train=True, batch_size=64)
    val_loader = get_loader(is_train=False, batch_size=64)
    criterion = CircleLoss(similarity='cos')

    for img, label in tqdm(train_loader):
        model.zero_grad()
        pred = model(img)
        loss = criterion(pred, label)
        loss.backward()
        optimizer.step()

    top = 0
    bot = 0
    for img, label in val_loader:
        pred = model(img)
        result = label.eq(pred.max(dim=1)[1])
        top += float(result.sum())
        bot += float(result.numel())

    print("Accuracy: {:.4f}".format(top / bot))


if __name__ == "__main__":
    main()

[qianjinhao]

Hello, @qianjinhao can you provide an example of testing validation on mnist?Because when I used mnist verification, the accuracy was almost zero, but I felt that there was no problem with your implementation of loss, which bothered me very much. The following is my test code.thanks

import torch
from torch import nn, Tensor
from torch.optim import SGD
from torch.utils.data import DataLoader
from torchvision.datasets import MNIST
from torchvision.transforms import ToTensor
from tqdm import tqdm

from CircleLoss import CircleLoss

class NormLinear(nn.Linear):
    def __init__(self, in_features: int, out_features: int) -> None:
        super(NormLinear, self).__init__(in_features, out_features, bias=False)

    def forward(self, inp: Tensor) -> Tensor:
        return nn.functional.linear(nn.functional.normalize(inp),
                                    nn.functional.normalize(self.weight))

def get_loader(is_train: bool, batch_size: int) -> DataLoader:
    return DataLoader(
        dataset=MNIST(root="./mnist_data", train=is_train, transform=ToTensor(), download=True),
        batch_size=batch_size,
        shuffle=is_train,
    )


class Model(nn.Module):
    def __init__(self) -> None:
        super(Model, self).__init__()
        self.feature_extractor = nn.Sequential(
            nn.Conv2d(in_channels=1, out_channels=8, kernel_size=5),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2),
            nn.Conv2d(in_channels=8, out_channels=16, kernel_size=5),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2),
            nn.Conv2d(in_channels=16, out_channels=32, kernel_size=3),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2),
        )
        self.linear = NormLinear(32, 10)

    def forward(self, inp: Tensor) -> Tensor:
        feature = self.feature_extractor(inp).mean(dim=[2, 3])
        return self.linear(feature)


def main():
    model = Model()
    optimizer = SGD(model.parameters(), lr=0.01, momentum=0.9)
    train_loader = get_loader(is_train=True, batch_size=64)
    val_loader = get_loader(is_train=False, batch_size=64)
    criterion = CircleLoss(similarity='cos')

    for img, label in tqdm(train_loader):
        model.zero_grad()
        pred = model(img)
        loss = criterion(pred, label)
        loss.backward()
        optimizer.step()

    top = 0
    bot = 0
    for img, label in val_loader:
        pred = model(img)
        result = label.eq(pred.max(dim=1)[1])
        top += float(result.sum())
        bot += float(result.numel())

    print("Accuracy: {:.4f}".format(top / bot))


if __name__ == "__main__":
    main()

I do feel for you - I've faced the same issue myself. I open this code with the attitude of learning, welcome to point out my code problems, I am still trying.

[zgsDream]

@qianjinhao Ok, thank you. I have one thing to confirm. Have you used your loss to test the classification task?This determines my search in the wrong direction

[qianjinhao]

@qianjinhao Ok, thank you. I have one thing to confirm. Have you used your loss to test the classification task?This determines my search in the wrong direction

I was going to see the effect at mnist, but I haven't been able to see the effect yet.When I asked the author, he said that the effect on the classification task might be uncertain, and it would be better to verify it on the data sets commonly used by metric learning.

[qianjinhao]

@qianjinhao Ok, thank you. I have one thing to confirm. Have you used your loss to test the classification task?This determines my search in the wrong direction

"pred.max(dim=1)[1]" is the problem I think where is it.Circle loss did not pass in the one-hot label, he will not learn this mapping relationship, if you want to test on mnist, you need to use a match test like Reid (find the smallest distance).

[TinyZeaMays]

@zgsDream

This example is designed for classification, but circle loss duels with problems of metric learning.

[moonfighting]

@qianjinhao Ok, thank you. I have one thing to confirm. Have you used your loss to test the classification task?This determines my search in the wrong direction
I think the main reason is that the loss didn't backward the gradient to the former layers，see my issue
#4