Diffuison_model_classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "49817375",
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import os\n",
    "\n",
    "import blobfile as bf\n",
    "import torch as th\n",
    "import torch\n",
    "import torch.distributed as dist\n",
    "import torch.nn.functional as F\n",
    "from torch.nn.parallel.distributed import DistributedDataParallel as DDP\n",
    "from torch.optim import AdamW\n",
    "\n",
    "from guided_diffusion import dist_util, logger\n",
    "from guided_diffusion.fp16_util import MixedPrecisionTrainer\n",
    "from guided_diffusion.image_datasets import load_data\n",
    "from guided_diffusion.resample import create_named_schedule_sampler\n",
    "from guided_diffusion.script_util import (\n",
    "    add_dict_to_argparser,\n",
    "    args_to_dict,\n",
    "    classifier_and_diffusion_defaults,\n",
    "    create_classifier_and_diffusion,\n",
    ")\n",
    "from guided_diffusion.train_util import parse_resume_step_from_filename, log_loss_dict\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1d9732da",
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import copy\n",
    "import sys\n",
    "import time\n",
    "from random import seed\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from matplotlib import animation\n",
    "from torch import optim\n",
    "from guided_diffusion.fp16_util import MixedPrecisionTrainer\n",
    "import dataset\n",
    "import evaluation\n",
    "from GaussianDiffusion import GaussianDiffusionModel, get_beta_schedule\n",
    "from helpers import *\n",
    "from UNet import UNetModel, update_ema_params\n",
    "from script_util import (\n",
    "    add_dict_to_argparser,\n",
    "    args_to_dict,\n",
    "    classifier_and_diffusion_defaults,\n",
    "    create_classifier_and_diffusion,\n",
    ")\n",
    "torch.cuda.empty_cache()\n",
    "\n",
    "# from train_util import  log_loss_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c73f6ebd",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "13b64998",
   "metadata": {},
   "outputs": [],
   "source": [
    "args_dict = {\n",
    "  \"image_size\": 256,\n",
    "  \"Batch_Size\": 16,\n",
    "  \"batch_size\": 16,\n",
    "  \"channels\": 3,\n",
    "  \"EPOCHS\": 4000,\n",
    "  \"iterations\":40000,\n",
    "  \"diffusion_steps\": 1000,\n",
    "  \"num_channels\": 192,\\\n",
    "  \"learn_sigma\":True,\n",
    "  \"val_data\": True,\n",
    "  \"eval_interval\": 1000,\n",
    "  \"save_interval\": 1000,\n",
    "  \"log_interval\": 10,\n",
    "  \"anneal_lr\": True,\n",
    "  \"microbatch\": -1,\n",
    "  \"use_kl\":False,\n",
    "  \"schedule_sampler\": \"uniform\",\n",
    "  \"resume_checkpoint\": False,\n",
    "  \"predict_xstart\":False,\n",
    "  \"rescale_timesteps\":False,\n",
    "  \"rescale_learned_sigmas\":False,\n",
    "  \"timestep_respacing\": \"\",\n",
    "  \"noise_schedule\": \"cosine\",\n",
    "  \"channel_mults\": \"\",\n",
    "  \"loss-type\": \"l2\",\n",
    "  \"loss_weight\": \"none\",\n",
    "  \"train_start\": True,\n",
    "  \"lr\": 1e-4,\n",
    "  \"learn_sigma \": True,\n",
    "  \"random_slice\": True,\n",
    "  \"sample_distance\": 800,\n",
    "  \"weight_decay\": 0.0,\n",
    "  \"save_imgs\": True,\n",
    "  \"save_vids\": True,\n",
    "  \"class_cond \": True,\n",
    "  \"use_fp16\": True,\n",
    "  \"use_scale_shift_norm\": True,\n",
    "  \"dropout\": 0.1,\n",
    "  \"attention_resolutions\": \"32,16,8\",\n",
    "  \"num_res_blocks\": 3,\n",
    "  \"resblock_updown\": True,\n",
    "  \"classifier_use_fp16\":False,\n",
    "  \"classifier_width\":128,\n",
    "  \"classifier_depth\": 2,\n",
    "  \"classifier_attention_resolutions\": \"32,16,8\",\n",
    "  \"classifier_use_scale_shift_norm\": True,\n",
    "  \"classifier_resblock_updown\": True,\n",
    "  \"classifier_pool\": \"attention\" ,\n",
    "  \"num_head_channels\": 64,\n",
    "  \"noised\": True,\n",
    "  \"noise_fn\": \"gauss\",\n",
    "  \"dataset\": \"mura\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "66a0c7e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "args = argparse.Namespace(**args_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "45019122",
   "metadata": {},
   "outputs": [],
   "source": [
    "# args."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bf862fe6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "92215f1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "ROOT_DIR= './'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "742bc9fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logging to C:\\Users\\Admin\\AppData\\Local\\Temp\\openai-2023-02-15-20-57-48-717999\n",
      "creating model and diffusion...\n",
      "creating data loader...\n",
      "creating optimizer...\n",
      "training classifier model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Admin\\anaconda3\\envs\\en_2\\lib\\site-packages\\torchvision\\transforms\\transforms.py:329: UserWarning: Argument 'interpolation' of type int is deprecated since 0.13 and will be removed in 0.15. Please use InterpolationMode enum.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------\n",
      "| grad_norm      | 145      |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 16       |\n",
      "| step           | 0        |\n",
      "| train_acc@1    | 0        |\n",
      "| train_acc@1_q0 | 0        |\n",
      "| train_acc@1_q1 | 0        |\n",
      "| train_acc@1_q2 | 0        |\n",
      "| train_acc@1_q3 | 0        |\n",
      "| train_acc@5    | 0        |\n",
      "| train_acc@5_q0 | 0        |\n",
      "| train_acc@5_q1 | 0        |\n",
      "| train_acc@5_q2 | 0        |\n",
      "| train_acc@5_q3 | 0        |\n",
      "| train_loss     | 6.78     |\n",
      "| train_loss_q0  | 6.75     |\n",
      "| train_loss_q1  | 6.77     |\n",
      "| train_loss_q2  | 6.79     |\n",
      "| train_loss_q3  | 6.84     |\n",
      "| val_acc@1      | 0.562    |\n",
      "| val_acc@1_q0   | 0.75     |\n",
      "| val_acc@1_q1   | 1        |\n",
      "| val_acc@1_q2   | 0.2      |\n",
      "| val_acc@1_q3   | 0.5      |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 4.89     |\n",
      "| val_loss_q0    | 4.65     |\n",
      "| val_loss_q1    | 4.23     |\n",
      "| val_loss_q2    | 5.4      |\n",
      "| val_loss_q3    | 4.97     |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 12.4     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 176      |\n",
      "| step           | 10       |\n",
      "| train_acc@1    | 0.65     |\n",
      "| train_acc@1_q0 | 0.625    |\n",
      "| train_acc@1_q1 | 0.684    |\n",
      "| train_acc@1_q2 | 0.743    |\n",
      "| train_acc@1_q3 | 0.574    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 3.18     |\n",
      "| train_loss_q0  | 3.36     |\n",
      "| train_loss_q1  | 3.05     |\n",
      "| train_loss_q2  | 2.96     |\n",
      "| train_loss_q3  | 3.32     |\n",
      "| val_acc@1      | 0.438    |\n",
      "| val_acc@1_q0   | 0.286    |\n",
      "| val_acc@1_q1   | 0.6      |\n",
      "| val_acc@1_q2   | 0.417    |\n",
      "| val_acc@1_q3   | 0.5      |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 2.94     |\n",
      "| val_loss_q0    | 3.38     |\n",
      "| val_loss_q1    | 2.26     |\n",
      "| val_loss_q2    | 3.07     |\n",
      "| val_loss_q3    | 2.8      |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 9.73     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 336      |\n",
      "| step           | 20       |\n",
      "| train_acc@1    | 0.594    |\n",
      "| train_acc@1_q0 | 0.514    |\n",
      "| train_acc@1_q1 | 0.611    |\n",
      "| train_acc@1_q2 | 0.619    |\n",
      "| train_acc@1_q3 | 0.622    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 1.31     |\n",
      "| train_loss_q0  | 1.53     |\n",
      "| train_loss_q1  | 1.21     |\n",
      "| train_loss_q2  | 1.18     |\n",
      "| train_loss_q3  | 1.33     |\n",
      "| val_acc@1      | 0.625    |\n",
      "| val_acc@1_q0   | 0.714    |\n",
      "| val_acc@1_q1   | 0.556    |\n",
      "| val_acc@1_q2   | 0.667    |\n",
      "| val_acc@1_q3   | 0.571    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.794    |\n",
      "| val_loss_q0    | 0.647    |\n",
      "| val_loss_q1    | 0.891    |\n",
      "| val_loss_q2    | 0.743    |\n",
      "| val_loss_q3    | 0.881    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 12.9     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 496      |\n",
      "| step           | 30       |\n",
      "| train_acc@1    | 0.494    |\n",
      "| train_acc@1_q0 | 0.617    |\n",
      "| train_acc@1_q1 | 0.462    |\n",
      "| train_acc@1_q2 | 0.371    |\n",
      "| train_acc@1_q3 | 0.487    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.811    |\n",
      "| train_loss_q0  | 0.707    |\n",
      "| train_loss_q1  | 0.89     |\n",
      "| train_loss_q2  | 0.877    |\n",
      "| train_loss_q3  | 0.798    |\n",
      "| val_acc@1      | 0.375    |\n",
      "| val_acc@1_q0   | 0.333    |\n",
      "| val_acc@1_q1   | 0.4      |\n",
      "| val_acc@1_q2   | 0.3      |\n",
      "| val_acc@1_q3   | 0.5      |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.821    |\n",
      "| val_loss_q0    | 0.838    |\n",
      "| val_loss_q1    | 0.848    |\n",
      "| val_loss_q2    | 0.843    |\n",
      "| val_loss_q3    | 0.758    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 6.52     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 656      |\n",
      "| step           | 40       |\n",
      "| train_acc@1    | 0.525    |\n",
      "| train_acc@1_q0 | 0.529    |\n",
      "| train_acc@1_q1 | 0.5      |\n",
      "| train_acc@1_q2 | 0.528    |\n",
      "| train_acc@1_q3 | 0.552    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.683    |\n",
      "| train_loss_q0  | 0.669    |\n",
      "| train_loss_q1  | 0.681    |\n",
      "| train_loss_q2  | 0.684    |\n",
      "| train_loss_q3  | 0.698    |\n",
      "| val_acc@1      | 0.375    |\n",
      "| val_acc@1_q0   | 0.4      |\n",
      "| val_acc@1_q1   | 0.375    |\n",
      "| val_acc@1_q2   | 0.6      |\n",
      "| val_acc@1_q3   | 0.111    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.732    |\n",
      "| val_loss_q0    | 0.901    |\n",
      "| val_loss_q1    | 0.629    |\n",
      "| val_loss_q2    | 0.565    |\n",
      "| val_loss_q3    | 0.917    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 7.84     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 816      |\n",
      "| step           | 50       |\n",
      "| train_acc@1    | 0.525    |\n",
      "| train_acc@1_q0 | 0.385    |\n",
      "| train_acc@1_q1 | 0.639    |\n",
      "| train_acc@1_q2 | 0.5      |\n",
      "| train_acc@1_q3 | 0.581    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.737    |\n",
      "| train_loss_q0  | 0.812    |\n",
      "| train_loss_q1  | 0.649    |\n",
      "| train_loss_q2  | 0.793    |\n",
      "| train_loss_q3  | 0.688    |\n",
      "| val_acc@1      | 0.5      |\n",
      "| val_acc@1_q0   | 0.364    |\n",
      "| val_acc@1_q1   | 0.5      |\n",
      "| val_acc@1_q2   | 0.8      |\n",
      "| val_acc@1_q3   | 0.2      |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.746    |\n",
      "| val_loss_q0    | 0.791    |\n",
      "| val_loss_q1    | 0.78     |\n",
      "| val_loss_q2    | 0.598    |\n",
      "| val_loss_q3    | 0.902    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 6.62     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 976      |\n",
      "| step           | 60       |\n",
      "| train_acc@1    | 0.562    |\n",
      "| train_acc@1_q0 | 0.528    |\n",
      "| train_acc@1_q1 | 0.667    |\n",
      "| train_acc@1_q2 | 0.457    |\n",
      "| train_acc@1_q3 | 0.619    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.681    |\n",
      "| train_loss_q0  | 0.736    |\n",
      "| train_loss_q1  | 0.615    |\n",
      "| train_loss_q2  | 0.739    |\n",
      "| train_loss_q3  | 0.626    |\n",
      "| val_acc@1      | 0.469    |\n",
      "| val_acc@1_q0   | 0.375    |\n",
      "| val_acc@1_q1   | 0.571    |\n",
      "| val_acc@1_q2   | 0.444    |\n",
      "| val_acc@1_q3   | 0.5      |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.839    |\n",
      "| val_loss_q0    | 0.961    |\n",
      "| val_loss_q1    | 0.771    |\n",
      "| val_loss_q2    | 0.765    |\n",
      "| val_loss_q3    | 0.86     |\n",
      "-----------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 6.92     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 1.14e+03 |\n",
      "| step           | 70       |\n",
      "| train_acc@1    | 0.613    |\n",
      "| train_acc@1_q0 | 0.667    |\n",
      "| train_acc@1_q1 | 0.556    |\n",
      "| train_acc@1_q2 | 0.571    |\n",
      "| train_acc@1_q3 | 0.647    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.707    |\n",
      "| train_loss_q0  | 0.666    |\n",
      "| train_loss_q1  | 0.763    |\n",
      "| train_loss_q2  | 0.738    |\n",
      "| train_loss_q3  | 0.669    |\n",
      "| val_acc@1      | 0.531    |\n",
      "| val_acc@1_q0   | 0.5      |\n",
      "| val_acc@1_q1   | 0.455    |\n",
      "| val_acc@1_q2   | 0.455    |\n",
      "| val_acc@1_q3   | 1        |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.702    |\n",
      "| val_loss_q0    | 0.729    |\n",
      "| val_loss_q1    | 0.682    |\n",
      "| val_loss_q2    | 0.777    |\n",
      "| val_loss_q3    | 0.511    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 4.01     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 1.3e+03  |\n",
      "| step           | 80       |\n",
      "| train_acc@1    | 0.625    |\n",
      "| train_acc@1_q0 | 0.479    |\n",
      "| train_acc@1_q1 | 0.75     |\n",
      "| train_acc@1_q2 | 0.674    |\n",
      "| train_acc@1_q3 | 0.649    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.671    |\n",
      "| train_loss_q0  | 0.75     |\n",
      "| train_loss_q1  | 0.602    |\n",
      "| train_loss_q2  | 0.63     |\n",
      "| train_loss_q3  | 0.674    |\n",
      "| val_acc@1      | 0.5      |\n",
      "| val_acc@1_q0   | 0.5      |\n",
      "| val_acc@1_q1   | 0.2      |\n",
      "| val_acc@1_q2   | 0.625    |\n",
      "| val_acc@1_q3   | 0.556    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.746    |\n",
      "| val_loss_q0    | 0.727    |\n",
      "| val_loss_q1    | 0.922    |\n",
      "| val_loss_q2    | 0.687    |\n",
      "| val_loss_q3    | 0.722    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 3.4      |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 1.46e+03 |\n",
      "| step           | 90       |\n",
      "| train_acc@1    | 0.613    |\n",
      "| train_acc@1_q0 | 0.587    |\n",
      "| train_acc@1_q1 | 0.676    |\n",
      "| train_acc@1_q2 | 0.667    |\n",
      "| train_acc@1_q3 | 0.545    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.67     |\n",
      "| train_loss_q0  | 0.678    |\n",
      "| train_loss_q1  | 0.621    |\n",
      "| train_loss_q2  | 0.67     |\n",
      "| train_loss_q3  | 0.699    |\n",
      "| val_acc@1      | 0.531    |\n",
      "| val_acc@1_q0   | 0.6      |\n",
      "| val_acc@1_q1   | 0.5      |\n",
      "| val_acc@1_q2   | 0.143    |\n",
      "| val_acc@1_q3   | 0.727    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.709    |\n",
      "| val_loss_q0    | 0.701    |\n",
      "| val_loss_q1    | 0.739    |\n",
      "| val_loss_q2    | 0.861    |\n",
      "| val_loss_q3    | 0.608    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 3.87     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 1.62e+03 |\n",
      "| step           | 100      |\n",
      "| train_acc@1    | 0.594    |\n",
      "| train_acc@1_q0 | 0.535    |\n",
      "| train_acc@1_q1 | 0.585    |\n",
      "| train_acc@1_q2 | 0.571    |\n",
      "| train_acc@1_q3 | 0.706    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.683    |\n",
      "| train_loss_q0  | 0.694    |\n",
      "| train_loss_q1  | 0.693    |\n",
      "| train_loss_q2  | 0.703    |\n",
      "| train_loss_q3  | 0.632    |\n",
      "| val_acc@1      | 0.531    |\n",
      "| val_acc@1_q0   | 0.4      |\n",
      "| val_acc@1_q1   | 0.714    |\n",
      "| val_acc@1_q2   | 0.444    |\n",
      "| val_acc@1_q3   | 0.667    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.719    |\n",
      "| val_loss_q0    | 0.809    |\n",
      "| val_loss_q1    | 0.588    |\n",
      "| val_loss_q2    | 0.826    |\n",
      "| val_loss_q3    | 0.562    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 6.79     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 1.78e+03 |\n",
      "| step           | 110      |\n",
      "| train_acc@1    | 0.494    |\n",
      "| train_acc@1_q0 | 0.457    |\n",
      "| train_acc@1_q1 | 0.575    |\n",
      "| train_acc@1_q2 | 0.476    |\n",
      "| train_acc@1_q3 | 0.469    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.701    |\n",
      "| train_loss_q0  | 0.726    |\n",
      "| train_loss_q1  | 0.652    |\n",
      "| train_loss_q2  | 0.702    |\n",
      "| train_loss_q3  | 0.724    |\n",
      "| val_acc@1      | 0.562    |\n",
      "| val_acc@1_q0   | 0.444    |\n",
      "| val_acc@1_q1   | 0.7      |\n",
      "| val_acc@1_q2   | 0.667    |\n",
      "| val_acc@1_q3   | 0.429    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.675    |\n",
      "| val_loss_q0    | 0.691    |\n",
      "| val_loss_q1    | 0.661    |\n",
      "| val_loss_q2    | 0.635    |\n",
      "| val_loss_q3    | 0.708    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 7.88     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 1.94e+03 |\n",
      "| step           | 120      |\n",
      "| train_acc@1    | 0.594    |\n",
      "| train_acc@1_q0 | 0.667    |\n",
      "| train_acc@1_q1 | 0.622    |\n",
      "| train_acc@1_q2 | 0.587    |\n",
      "| train_acc@1_q3 | 0.5      |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.686    |\n",
      "| train_loss_q0  | 0.651    |\n",
      "| train_loss_q1  | 0.666    |\n",
      "| train_loss_q2  | 0.669    |\n",
      "| train_loss_q3  | 0.76     |\n",
      "| val_acc@1      | 0.406    |\n",
      "| val_acc@1_q0   | 0.3      |\n",
      "| val_acc@1_q1   | 0.143    |\n",
      "| val_acc@1_q2   | 0.625    |\n",
      "| val_acc@1_q3   | 0.571    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.876    |\n",
      "| val_loss_q0    | 0.935    |\n",
      "| val_loss_q1    | 1.12     |\n",
      "| val_loss_q2    | 0.71     |\n",
      "| val_loss_q3    | 0.744    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 8.74     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 2.1e+03  |\n",
      "| step           | 130      |\n",
      "| train_acc@1    | 0.469    |\n",
      "| train_acc@1_q0 | 0.531    |\n",
      "| train_acc@1_q1 | 0.421    |\n",
      "| train_acc@1_q2 | 0.574    |\n",
      "| train_acc@1_q3 | 0.349    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.706    |\n",
      "| train_loss_q0  | 0.674    |\n",
      "| train_loss_q1  | 0.713    |\n",
      "| train_loss_q2  | 0.683    |\n",
      "| train_loss_q3  | 0.748    |\n",
      "| val_acc@1      | 0.719    |\n",
      "| val_acc@1_q0   | 0.875    |\n",
      "| val_acc@1_q1   | 0.625    |\n",
      "| val_acc@1_q2   | 0.833    |\n",
      "| val_acc@1_q3   | 0.6      |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.599    |\n",
      "| val_loss_q0    | 0.524    |\n",
      "| val_loss_q1    | 0.63     |\n",
      "| val_loss_q2    | 0.576    |\n",
      "| val_loss_q3    | 0.649    |\n",
      "-----------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 5.42     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 2.26e+03 |\n",
      "| step           | 140      |\n",
      "| train_acc@1    | 0.6      |\n",
      "| train_acc@1_q0 | 0.55     |\n",
      "| train_acc@1_q1 | 0.625    |\n",
      "| train_acc@1_q2 | 0.615    |\n",
      "| train_acc@1_q3 | 0.609    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.706    |\n",
      "| train_loss_q0  | 0.772    |\n",
      "| train_loss_q1  | 0.669    |\n",
      "| train_loss_q2  | 0.668    |\n",
      "| train_loss_q3  | 0.708    |\n",
      "| val_acc@1      | 0.406    |\n",
      "| val_acc@1_q0   | 0.455    |\n",
      "| val_acc@1_q1   | 0.3      |\n",
      "| val_acc@1_q2   | 0.5      |\n",
      "| val_acc@1_q3   | 0.429    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.807    |\n",
      "| val_loss_q0    | 0.78     |\n",
      "| val_loss_q1    | 0.907    |\n",
      "| val_loss_q2    | 0.762    |\n",
      "| val_loss_q3    | 0.733    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 4.24     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 2.42e+03 |\n",
      "| step           | 150      |\n",
      "| train_acc@1    | 0.631    |\n",
      "| train_acc@1_q0 | 0.653    |\n",
      "| train_acc@1_q1 | 0.455    |\n",
      "| train_acc@1_q2 | 0.683    |\n",
      "| train_acc@1_q3 | 0.703    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.664    |\n",
      "| train_loss_q0  | 0.654    |\n",
      "| train_loss_q1  | 0.706    |\n",
      "| train_loss_q2  | 0.639    |\n",
      "| train_loss_q3  | 0.669    |\n",
      "| val_acc@1      | 0.469    |\n",
      "| val_acc@1_q0   | 0.4      |\n",
      "| val_acc@1_q1   | 0.625    |\n",
      "| val_acc@1_q2   | 0.429    |\n",
      "| val_acc@1_q3   | 0.429    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.779    |\n",
      "| val_loss_q0    | 0.731    |\n",
      "| val_loss_q1    | 0.764    |\n",
      "| val_loss_q2    | 0.794    |\n",
      "| val_loss_q3    | 0.851    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 7.12     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 2.58e+03 |\n",
      "| step           | 160      |\n",
      "| train_acc@1    | 0.487    |\n",
      "| train_acc@1_q0 | 0.512    |\n",
      "| train_acc@1_q1 | 0.45     |\n",
      "| train_acc@1_q2 | 0.489    |\n",
      "| train_acc@1_q3 | 0.5      |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.734    |\n",
      "| train_loss_q0  | 0.697    |\n",
      "| train_loss_q1  | 0.75     |\n",
      "| train_loss_q2  | 0.761    |\n",
      "| train_loss_q3  | 0.723    |\n",
      "| val_acc@1      | 0.406    |\n",
      "| val_acc@1_q0   | 0.545    |\n",
      "| val_acc@1_q1   | 0.364    |\n",
      "| val_acc@1_q2   | 0.4      |\n",
      "| val_acc@1_q3   | 0.2      |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.745    |\n",
      "| val_loss_q0    | 0.707    |\n",
      "| val_loss_q1    | 0.778    |\n",
      "| val_loss_q2    | 0.69     |\n",
      "| val_loss_q3    | 0.813    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 8.17     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 2.74e+03 |\n",
      "| step           | 170      |\n",
      "| train_acc@1    | 0.494    |\n",
      "| train_acc@1_q0 | 0.4      |\n",
      "| train_acc@1_q1 | 0.523    |\n",
      "| train_acc@1_q2 | 0.605    |\n",
      "| train_acc@1_q3 | 0.447    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.742    |\n",
      "| train_loss_q0  | 0.771    |\n",
      "| train_loss_q1  | 0.747    |\n",
      "| train_loss_q2  | 0.72     |\n",
      "| train_loss_q3  | 0.726    |\n",
      "| val_acc@1      | 0.562    |\n",
      "| val_acc@1_q0   | 0.615    |\n",
      "| val_acc@1_q1   | 0.8      |\n",
      "| val_acc@1_q2   | 0.375    |\n",
      "| val_acc@1_q3   | 0.5      |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.722    |\n",
      "| val_loss_q0    | 0.699    |\n",
      "| val_loss_q1    | 0.565    |\n",
      "| val_loss_q2    | 0.842    |\n",
      "| val_loss_q3    | 0.741    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 3.72     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 2.9e+03  |\n",
      "| step           | 180      |\n",
      "| train_acc@1    | 0.525    |\n",
      "| train_acc@1_q0 | 0.579    |\n",
      "| train_acc@1_q1 | 0.541    |\n",
      "| train_acc@1_q2 | 0.523    |\n",
      "| train_acc@1_q3 | 0.463    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.695    |\n",
      "| train_loss_q0  | 0.674    |\n",
      "| train_loss_q1  | 0.7      |\n",
      "| train_loss_q2  | 0.693    |\n",
      "| train_loss_q3  | 0.711    |\n",
      "| val_acc@1      | 0.438    |\n",
      "| val_acc@1_q0   | 0.444    |\n",
      "| val_acc@1_q1   | 0.5      |\n",
      "| val_acc@1_q2   | 0.2      |\n",
      "| val_acc@1_q3   | 0.5      |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.715    |\n",
      "| val_loss_q0    | 0.697    |\n",
      "| val_loss_q1    | 0.737    |\n",
      "| val_loss_q2    | 0.735    |\n",
      "| val_loss_q3    | 0.704    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 4.88     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 3.06e+03 |\n",
      "| step           | 190      |\n",
      "| train_acc@1    | 0.5      |\n",
      "| train_acc@1_q0 | 0.512    |\n",
      "| train_acc@1_q1 | 0.5      |\n",
      "| train_acc@1_q2 | 0.643    |\n",
      "| train_acc@1_q3 | 0.341    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.707    |\n",
      "| train_loss_q0  | 0.71     |\n",
      "| train_loss_q1  | 0.688    |\n",
      "| train_loss_q2  | 0.669    |\n",
      "| train_loss_q3  | 0.76     |\n",
      "| val_acc@1      | 0.5      |\n",
      "| val_acc@1_q0   | 0.5      |\n",
      "| val_acc@1_q1   | 0.286    |\n",
      "| val_acc@1_q2   | 0.6      |\n",
      "| val_acc@1_q3   | 0.571    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.705    |\n",
      "| val_loss_q0    | 0.693    |\n",
      "| val_loss_q1    | 0.751    |\n",
      "| val_loss_q2    | 0.708    |\n",
      "| val_loss_q3    | 0.67     |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 7.48     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 3.22e+03 |\n",
      "| step           | 200      |\n",
      "| train_acc@1    | 0.581    |\n",
      "| train_acc@1_q0 | 0.463    |\n",
      "| train_acc@1_q1 | 0.7      |\n",
      "| train_acc@1_q2 | 0.571    |\n",
      "| train_acc@1_q3 | 0.588    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.708    |\n",
      "| train_loss_q0  | 0.735    |\n",
      "| train_loss_q1  | 0.648    |\n",
      "| train_loss_q2  | 0.739    |\n",
      "| train_loss_q3  | 0.717    |\n",
      "| val_acc@1      | 0.344    |\n",
      "| val_acc@1_q0   | 0.333    |\n",
      "| val_acc@1_q1   | 0.167    |\n",
      "| val_acc@1_q2   | 0.333    |\n",
      "| val_acc@1_q3   | 0.455    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.779    |\n",
      "| val_loss_q0    | 0.771    |\n",
      "| val_loss_q1    | 0.827    |\n",
      "| val_loss_q2    | 0.808    |\n",
      "| val_loss_q3    | 0.733    |\n",
      "-----------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 6.56     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 3.38e+03 |\n",
      "| step           | 210      |\n",
      "| train_acc@1    | 0.487    |\n",
      "| train_acc@1_q0 | 0.436    |\n",
      "| train_acc@1_q1 | 0.487    |\n",
      "| train_acc@1_q2 | 0.625    |\n",
      "| train_acc@1_q3 | 0.405    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.713    |\n",
      "| train_loss_q0  | 0.714    |\n",
      "| train_loss_q1  | 0.714    |\n",
      "| train_loss_q2  | 0.681    |\n",
      "| train_loss_q3  | 0.743    |\n",
      "| val_acc@1      | 0.438    |\n",
      "| val_acc@1_q0   | 0.167    |\n",
      "| val_acc@1_q1   | 0.429    |\n",
      "| val_acc@1_q2   | 0.417    |\n",
      "| val_acc@1_q3   | 0.714    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.722    |\n",
      "| val_loss_q0    | 0.763    |\n",
      "| val_loss_q1    | 0.694    |\n",
      "| val_loss_q2    | 0.735    |\n",
      "| val_loss_q3    | 0.692    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 4.53     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 3.54e+03 |\n",
      "| step           | 220      |\n",
      "| train_acc@1    | 0.594    |\n",
      "| train_acc@1_q0 | 0.649    |\n",
      "| train_acc@1_q1 | 0.531    |\n",
      "| train_acc@1_q2 | 0.667    |\n",
      "| train_acc@1_q3 | 0.561    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.687    |\n",
      "| train_loss_q0  | 0.615    |\n",
      "| train_loss_q1  | 0.746    |\n",
      "| train_loss_q2  | 0.682    |\n",
      "| train_loss_q3  | 0.687    |\n",
      "| val_acc@1      | 0.5      |\n",
      "| val_acc@1_q0   | 0.778    |\n",
      "| val_acc@1_q1   | 0.444    |\n",
      "| val_acc@1_q2   | 0.286    |\n",
      "| val_acc@1_q3   | 0.429    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.763    |\n",
      "| val_loss_q0    | 0.526    |\n",
      "| val_loss_q1    | 0.809    |\n",
      "| val_loss_q2    | 0.952    |\n",
      "| val_loss_q3    | 0.818    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 6.12     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 3.7e+03  |\n",
      "| step           | 230      |\n",
      "| train_acc@1    | 0.487    |\n",
      "| train_acc@1_q0 | 0.415    |\n",
      "| train_acc@1_q1 | 0.353    |\n",
      "| train_acc@1_q2 | 0.551    |\n",
      "| train_acc@1_q3 | 0.611    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.69     |\n",
      "| train_loss_q0  | 0.727    |\n",
      "| train_loss_q1  | 0.757    |\n",
      "| train_loss_q2  | 0.655    |\n",
      "| train_loss_q3  | 0.634    |\n",
      "| val_acc@1      | 0.469    |\n",
      "| val_acc@1_q0   | 0.6      |\n",
      "| val_acc@1_q1   | 0.375    |\n",
      "| val_acc@1_q2   | 0.5      |\n",
      "| val_acc@1_q3   | 0.333    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.703    |\n",
      "| val_loss_q0    | 0.677    |\n",
      "| val_loss_q1    | 0.726    |\n",
      "| val_loss_q2    | 0.688    |\n",
      "| val_loss_q3    | 0.734    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 7.67     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 3.86e+03 |\n",
      "| step           | 240      |\n",
      "| train_acc@1    | 0.55     |\n",
      "| train_acc@1_q0 | 0.35     |\n",
      "| train_acc@1_q1 | 0.696    |\n",
      "| train_acc@1_q2 | 0.535    |\n",
      "| train_acc@1_q3 | 0.613    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.727    |\n",
      "| train_loss_q0  | 0.846    |\n",
      "| train_loss_q1  | 0.617    |\n",
      "| train_loss_q2  | 0.776    |\n",
      "| train_loss_q3  | 0.671    |\n",
      "| val_acc@1      | 0.531    |\n",
      "| val_acc@1_q0   | 0.5      |\n",
      "| val_acc@1_q1   | 0.143    |\n",
      "| val_acc@1_q2   | 0.7      |\n",
      "| val_acc@1_q3   | 0.714    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.751    |\n",
      "| val_loss_q0    | 0.771    |\n",
      "| val_loss_q1    | 1.13     |\n",
      "| val_loss_q2    | 0.598    |\n",
      "| val_loss_q3    | 0.572    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 6        |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 4.02e+03 |\n",
      "| step           | 250      |\n",
      "| train_acc@1    | 0.431    |\n",
      "| train_acc@1_q0 | 0.562    |\n",
      "| train_acc@1_q1 | 0.465    |\n",
      "| train_acc@1_q2 | 0.289    |\n",
      "| train_acc@1_q3 | 0.426    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.714    |\n",
      "| train_loss_q0  | 0.678    |\n",
      "| train_loss_q1  | 0.705    |\n",
      "| train_loss_q2  | 0.746    |\n",
      "| train_loss_q3  | 0.723    |\n",
      "| val_acc@1      | 0.438    |\n",
      "| val_acc@1_q0   | 0.5      |\n",
      "| val_acc@1_q1   | 0.2      |\n",
      "| val_acc@1_q2   | 0.667    |\n",
      "| val_acc@1_q3   | 0.556    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.72     |\n",
      "| val_loss_q0    | 0.713    |\n",
      "| val_loss_q1    | 0.751    |\n",
      "| val_loss_q2    | 0.613    |\n",
      "| val_loss_q3    | 0.728    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 6.66     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 4.18e+03 |\n",
      "| step           | 260      |\n",
      "| train_acc@1    | 0.575    |\n",
      "| train_acc@1_q0 | 0.517    |\n",
      "| train_acc@1_q1 | 0.489    |\n",
      "| train_acc@1_q2 | 0.6      |\n",
      "| train_acc@1_q3 | 0.673    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.701    |\n",
      "| train_loss_q0  | 0.706    |\n",
      "| train_loss_q1  | 0.763    |\n",
      "| train_loss_q2  | 0.691    |\n",
      "| train_loss_q3  | 0.644    |\n",
      "| val_acc@1      | 0.5      |\n",
      "| val_acc@1_q0   | 0.8      |\n",
      "| val_acc@1_q1   | 0.333    |\n",
      "| val_acc@1_q2   | 0.625    |\n",
      "| val_acc@1_q3   | 0.4      |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.752    |\n",
      "| val_loss_q0    | 0.553    |\n",
      "| val_loss_q1    | 0.881    |\n",
      "| val_loss_q2    | 0.665    |\n",
      "| val_loss_q3    | 0.804    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 6.92     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 4.34e+03 |\n",
      "| step           | 270      |\n",
      "| train_acc@1    | 0.575    |\n",
      "| train_acc@1_q0 | 0.632    |\n",
      "| train_acc@1_q1 | 0.426    |\n",
      "| train_acc@1_q2 | 0.63     |\n",
      "| train_acc@1_q3 | 0.655    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.686    |\n",
      "| train_loss_q0  | 0.657    |\n",
      "| train_loss_q1  | 0.726    |\n",
      "| train_loss_q2  | 0.671    |\n",
      "| train_loss_q3  | 0.679    |\n",
      "| val_acc@1      | 0.594    |\n",
      "| val_acc@1_q0   | 0.714    |\n",
      "| val_acc@1_q1   | 0.6      |\n",
      "| val_acc@1_q2   | 0.556    |\n",
      "| val_acc@1_q3   | 0.545    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.72     |\n",
      "| val_loss_q0    | 0.662    |\n",
      "| val_loss_q1    | 0.587    |\n",
      "| val_loss_q2    | 0.754    |\n",
      "| val_loss_q3    | 0.789    |\n",
      "-----------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 6.11     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 4.5e+03  |\n",
      "| step           | 280      |\n",
      "| train_acc@1    | 0.612    |\n",
      "| train_acc@1_q0 | 0.703    |\n",
      "| train_acc@1_q1 | 0.564    |\n",
      "| train_acc@1_q2 | 0.548    |\n",
      "| train_acc@1_q3 | 0.643    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.676    |\n",
      "| train_loss_q0  | 0.586    |\n",
      "| train_loss_q1  | 0.708    |\n",
      "| train_loss_q2  | 0.737    |\n",
      "| train_loss_q3  | 0.667    |\n",
      "| val_acc@1      | 0.438    |\n",
      "| val_acc@1_q0   | 0.5      |\n",
      "| val_acc@1_q1   | 0.571    |\n",
      "| val_acc@1_q2   | 0.3      |\n",
      "| val_acc@1_q3   | 0.429    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.827    |\n",
      "| val_loss_q0    | 0.813    |\n",
      "| val_loss_q1    | 0.722    |\n",
      "| val_loss_q2    | 0.912    |\n",
      "| val_loss_q3    | 0.829    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 4.93     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 4.66e+03 |\n",
      "| step           | 290      |\n",
      "| train_acc@1    | 0.594    |\n",
      "| train_acc@1_q0 | 0.556    |\n",
      "| train_acc@1_q1 | 0.61     |\n",
      "| train_acc@1_q2 | 0.634    |\n",
      "| train_acc@1_q3 | 0.571    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.689    |\n",
      "| train_loss_q0  | 0.689    |\n",
      "| train_loss_q1  | 0.695    |\n",
      "| train_loss_q2  | 0.691    |\n",
      "| train_loss_q3  | 0.682    |\n",
      "| val_acc@1      | 0.719    |\n",
      "| val_acc@1_q0   | 0.818    |\n",
      "| val_acc@1_q1   | 0.833    |\n",
      "| val_acc@1_q2   | 0.556    |\n",
      "| val_acc@1_q3   | 0.667    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.657    |\n",
      "| val_loss_q0    | 0.656    |\n",
      "| val_loss_q1    | 0.633    |\n",
      "| val_loss_q2    | 0.636    |\n",
      "| val_loss_q3    | 0.713    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 7.76     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 4.82e+03 |\n",
      "| step           | 300      |\n",
      "| train_acc@1    | 0.669    |\n",
      "| train_acc@1_q0 | 0.74     |\n",
      "| train_acc@1_q1 | 0.683    |\n",
      "| train_acc@1_q2 | 0.576    |\n",
      "| train_acc@1_q3 | 0.639    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.671    |\n",
      "| train_loss_q0  | 0.604    |\n",
      "| train_loss_q1  | 0.669    |\n",
      "| train_loss_q2  | 0.756    |\n",
      "| train_loss_q3  | 0.689    |\n",
      "| val_acc@1      | 0.656    |\n",
      "| val_acc@1_q0   | 0.778    |\n",
      "| val_acc@1_q1   | 0.7      |\n",
      "| val_acc@1_q2   | 0.333    |\n",
      "| val_acc@1_q3   | 0.714    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.668    |\n",
      "| val_loss_q0    | 0.545    |\n",
      "| val_loss_q1    | 0.591    |\n",
      "| val_loss_q2    | 0.978    |\n",
      "| val_loss_q3    | 0.671    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "-----------------------------\n",
      "| grad_norm      | 5.76     |\n",
      "| param_norm     | 160      |\n",
      "| samples        | 4.98e+03 |\n",
      "| step           | 310      |\n",
      "| train_acc@1    | 0.606    |\n",
      "| train_acc@1_q0 | 0.633    |\n",
      "| train_acc@1_q1 | 0.636    |\n",
      "| train_acc@1_q2 | 0.448    |\n",
      "| train_acc@1_q3 | 0.658    |\n",
      "| train_acc@5    | 1        |\n",
      "| train_acc@5_q0 | 1        |\n",
      "| train_acc@5_q1 | 1        |\n",
      "| train_acc@5_q2 | 1        |\n",
      "| train_acc@5_q3 | 1        |\n",
      "| train_loss     | 0.69     |\n",
      "| train_loss_q0  | 0.651    |\n",
      "| train_loss_q1  | 0.697    |\n",
      "| train_loss_q2  | 0.775    |\n",
      "| train_loss_q3  | 0.667    |\n",
      "| val_acc@1      | 0.656    |\n",
      "| val_acc@1_q0   | 0.889    |\n",
      "| val_acc@1_q1   | 0.714    |\n",
      "| val_acc@1_q2   | 0.6      |\n",
      "| val_acc@1_q3   | 0.333    |\n",
      "| val_acc@5      | 1        |\n",
      "| val_acc@5_q0   | 1        |\n",
      "| val_acc@5_q1   | 1        |\n",
      "| val_acc@5_q2   | 1        |\n",
      "| val_acc@5_q3   | 1        |\n",
      "| val_loss       | 0.674    |\n",
      "| val_loss_q0    | 0.665    |\n",
      "| val_loss_q1    | 0.643    |\n",
      "| val_loss_q2    | 0.695    |\n",
      "| val_loss_q3    | 0.689    |\n",
      "-----------------------------\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n",
      "saving model...\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "[enforce fail at C:\\actions-runner\\_work\\pytorch\\pytorch\\builder\\windows\\pytorch\\caffe2\\serialize\\inline_container.cc:325] . unexpected pos 128958528 vs 128958420",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "File \u001b[1;32m~\\anaconda3\\envs\\en_2\\lib\\site-packages\\torch\\serialization.py:436\u001b[0m, in \u001b[0;36msave\u001b[1;34m(obj, f, pickle_module, pickle_protocol, _use_new_zipfile_serialization)\u001b[0m\n\u001b[0;32m    435\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m _open_zipfile_writer(f) \u001b[38;5;28;01mas\u001b[39;00m opened_zipfile:\n\u001b[1;32m--> 436\u001b[0m     \u001b[43m_save\u001b[49m\u001b[43m(\u001b[49m\u001b[43mobj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mopened_zipfile\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpickle_module\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpickle_protocol\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    437\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\en_2\\lib\\site-packages\\torch\\serialization.py:663\u001b[0m, in \u001b[0;36m_save\u001b[1;34m(obj, zip_file, pickle_module, pickle_protocol)\u001b[0m\n\u001b[0;32m    662\u001b[0m num_bytes \u001b[38;5;241m=\u001b[39m storage\u001b[38;5;241m.\u001b[39mnbytes()\n\u001b[1;32m--> 663\u001b[0m \u001b[43mzip_file\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrite_record\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstorage\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdata_ptr\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_bytes\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[1;31mRuntimeError\u001b[0m: [enforce fail at C:\\actions-runner\\_work\\pytorch\\pytorch\\builder\\windows\\pytorch\\caffe2\\serialize\\inline_container.cc:464] . PytorchStreamWriter failed writing file data/481: file write failed",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[9], line 183\u001b[0m\n\u001b[0;32m    179\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m parser\n\u001b[0;32m    182\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;18m__name__\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m--> 183\u001b[0m     \u001b[43mmain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[1;32mIn[9], line 120\u001b[0m, in \u001b[0;36mmain\u001b[1;34m()\u001b[0m\n\u001b[0;32m    118\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m (step \u001b[38;5;241m+\u001b[39m resume_step) \u001b[38;5;241m%\u001b[39m args\u001b[38;5;241m.\u001b[39msave_interval:\n\u001b[0;32m    119\u001b[0m             logger\u001b[38;5;241m.\u001b[39mlog(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msaving model...\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m--> 120\u001b[0m             \u001b[43msave_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmp_trainer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mopt\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstep\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mresume_step\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    122\u001b[0m \u001b[38;5;66;03m#     if dist.get_rank() == 0:\u001b[39;00m\n\u001b[0;32m    123\u001b[0m     logger\u001b[38;5;241m.\u001b[39mlog(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msaving model...\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "Cell \u001b[1;32mIn[9], line 139\u001b[0m, in \u001b[0;36msave_model\u001b[1;34m(mp_trainer, opt, step)\u001b[0m\n\u001b[0;32m    134\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21msave_model\u001b[39m(mp_trainer, opt, step):\n\u001b[0;32m    135\u001b[0m     th\u001b[38;5;241m.\u001b[39msave(\n\u001b[0;32m    136\u001b[0m         mp_trainer\u001b[38;5;241m.\u001b[39mmaster_params_to_state_dict(mp_trainer\u001b[38;5;241m.\u001b[39mmaster_params),\n\u001b[0;32m    137\u001b[0m         os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(logger\u001b[38;5;241m.\u001b[39mget_dir(), \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmodel\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mstep\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m06d\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.pt\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[0;32m    138\u001b[0m     )\n\u001b[1;32m--> 139\u001b[0m     \u001b[43mth\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msave\u001b[49m\u001b[43m(\u001b[49m\u001b[43mopt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstate_dict\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpath\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjoin\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlogger\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_dir\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mopt\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mstep\u001b[49m\u001b[38;5;132;43;01m:\u001b[39;49;00m\u001b[38;5;124;43m06d\u001b[39;49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m.pt\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\en_2\\lib\\site-packages\\torch\\serialization.py:437\u001b[0m, in \u001b[0;36msave\u001b[1;34m(obj, f, pickle_module, pickle_protocol, _use_new_zipfile_serialization)\u001b[0m\n\u001b[0;32m    435\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m _open_zipfile_writer(f) \u001b[38;5;28;01mas\u001b[39;00m opened_zipfile:\n\u001b[0;32m    436\u001b[0m         _save(obj, opened_zipfile, pickle_module, pickle_protocol)\n\u001b[1;32m--> 437\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[0;32m    438\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    439\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m _open_file_like(f, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwb\u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m opened_file:\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\en_2\\lib\\site-packages\\torch\\serialization.py:290\u001b[0m, in \u001b[0;36m_open_zipfile_writer_file.__exit__\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m    289\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__exit__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m--> 290\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfile_like\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrite_end_of_file\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[1;31mRuntimeError\u001b[0m: [enforce fail at C:\\actions-runner\\_work\\pytorch\\pytorch\\builder\\windows\\pytorch\\caffe2\\serialize\\inline_container.cc:325] . unexpected pos 128958528 vs 128958420"
     ]
    }
   ],
   "source": [
    "def main():\n",
    "#     args = create_argparser().parse_args()\n",
    "\n",
    "#     dist_util.setup_dist()\n",
    "    logger.configure()\n",
    "\n",
    "    logger.log(\"creating model and diffusion...\")\n",
    "    model, diffusion = create_classifier_and_diffusion(args_dict\n",
    "    )\n",
    "    model.to(device)\n",
    "    if args.noised:\n",
    "        schedule_sampler = create_named_schedule_sampler(\n",
    "            args.schedule_sampler, diffusion\n",
    "        )\n",
    "\n",
    "    resume_step = 0\n",
    "    if args.resume_checkpoint:\n",
    "        resume_step = parse_resume_step_from_filename(args.resume_checkpoint)\n",
    "#         if dist.get_rank() == 0:\n",
    "        logger.log(\n",
    "            f\"loading model from checkpoint: {args.resume_checkpoint}... at {resume_step} step\"\n",
    "        )\n",
    "        model.load_state_dict(\n",
    "            dist_util.load_state_dict(\n",
    "                args.resume_checkpoint, map_location=device\n",
    "            )\n",
    "        )\n",
    "\n",
    "    # Needed for creating correct EMAs and fp16 parameters.\n",
    "#     dist_util.sync_params(model.parameters())\n",
    "\n",
    "    mp_trainer = MixedPrecisionTrainer(\n",
    "        model=model, use_fp16=args.classifier_use_fp16, initial_lg_loss_scale=16.0\n",
    "    )\n",
    "\n",
    "#     model = DDP(\n",
    "#         model,\n",
    "#         device_ids=[dist_util.dev()],\n",
    "#         output_device=device(),\n",
    "#         broadcast_buffers=False,\n",
    "#         bucket_cap_mb=128,\n",
    "#         find_unused_parameters=False,\n",
    "#     )\n",
    "\n",
    "    logger.log(\"creating data loader...\")\n",
    "    training_dataset, testing_dataset = dataset.init_datasets(ROOT_DIR, args_dict)\n",
    "    data = dataset.init_dataset_loader(training_dataset, args_dict)\n",
    "    if args.val_data:\n",
    "        val_data = dataset.init_dataset_loader(testing_dataset, args_dict)\n",
    "    else:\n",
    "        val_data = None\n",
    "\n",
    "    logger.log(f\"creating optimizer...\")\n",
    "    opt = AdamW(mp_trainer.master_params, lr=args.lr, weight_decay=args.weight_decay)\n",
    "    if args.resume_checkpoint:\n",
    "        opt_checkpoint = bf.join(\n",
    "            bf.dirname(args.resume_checkpoint), f\"opt{resume_step:06}.pt\"\n",
    "        )\n",
    "        logger.log(f\"loading optimizer state from checkpoint: {opt_checkpoint}\")\n",
    "        opt.load_state_dict(\n",
    "            dist_util.load_state_dict(opt_checkpoint, map_location=device)\n",
    "        )\n",
    "\n",
    "    logger.log(\"training classifier model...\")\n",
    "\n",
    "    def forward_backward_log(data_loader, prefix=\"train\"):\n",
    "        extra = next(data_loader)\n",
    "        batch = extra[\"image\"]\n",
    "        labels = extra[\"label\"].to(device)\n",
    "\n",
    "        batch = batch        extra = next(data_loader)\n",
    "        batch = extra[\"image\"]\n",
    "        labels = extra[\"label\"].to(device)\n",
    "\n",
    "        # Noisy images\n",
    "        if args.noised:\n",
    "            t, _ = schedule_sampler.sample(batch.shape[0], device)\n",
    "            batch = diffusion.q_sample(batch, t)\n",
    "        else:\n",
    "            t = th.zeros(batch.shape[0], dtype=th.long, device=device)\n",
    "\n",
    "        for i, (sub_batch, sub_labels, sub_t) in enumerate(\n",
    "            split_microbatches(args.microbatch, batch, labels, t)\n",
    "        ):\n",
    "            logits = model(sub_batch, timesteps=sub_t)\n",
    "            loss = F.cross_entropy(logits, sub_labels, reduction=\"none\")\n",
    "\n",
    "            losses = {}\n",
    "            losses[f\"{prefix}_loss\"] = loss.detach()\n",
    "            losses[f\"{prefix}_acc@1\"] = compute_top_k(\n",
    "                logits, sub_labels, k=1, reduction=\"none\"\n",
    "            )\n",
    "            losses[f\"{prefix}_acc@5\"] = compute_top_k(\n",
    "                logits, sub_labels, k=5, reduction=\"none\"\n",
    "            )\n",
    "            log_loss_dict(diffusion, sub_t, losses)\n",
    "            del losses\n",
    "            loss = loss.mean()\n",
    "            if loss.requires_grad:\n",
    "                if i == 0:\n",
    "                    mp_trainer.zero_grad()\n",
    "                mp_trainer.backward(loss * len(sub_batch) / len(batch))\n",
    "\n",
    "    for step in tqdm(range(args.iterations - resume_step)):\n",
    "        logger.logkv(\"step\", step + resume_step)\n",
    "        logger.logkv(\n",
    "            \"samples\",\n",
    "            (step + resume_step + 1) * args.batch_size,\n",
    "        )\n",
    "        if args.anneal_lr:\n",
    "            set_annealed_lr(opt, args.lr, (step + resume_step) / args.iterations)\n",
    "        forward_backward_log(data)\n",
    "        mp_trainer.optimize(opt)\n",
    "        if val_data is not None and not step % args.eval_interval:\n",
    "            with th.no_grad():\n",
    "                model.eval()\n",
    "                forward_backward_log(val_data, prefix=\"val\")\n",
    "                model.train()\n",
    "        if not step % args.log_interval:\n",
    "            logger.dumpkvs()\n",
    "        if (step + resume_step) % args.save_interval:\n",
    "            logger.log(\"saving model...\")\n",
    "            save_model(mp_trainer, opt, step + resume_step)\n",
    "\n",
    "#     if dist.get_rank() == 0:\n",
    "    logger.log(\"saving model...\")\n",
    "    save_model(mp_trainer, opt, step + resume_step)\n",
    "#     dist.barrier()\n",
    "\n",
    "\n",
    "def set_annealed_lr(opt, base_lr, frac_done):\n",
    "    lr = base_lr * (1 - frac_done)\n",
    "    for param_group in opt.param_groups:\n",
    "        param_group[\"lr\"] = lr\n",
    "\n",
    "\n",
    "def save_model(mp_trainer, opt, step):\n",
    "    th.save(\n",
    "        mp_trainer.master_params_to_state_dict(mp_trainer.master_params),\n",
    "        os.path.join(logger.get_dir(), f\"model{step:06d}.pt\"),\n",
    "    )\n",
    "    th.save(opt.state_dict(), os.path.join(logger.get_dir(), f\"opt{step:06d}.pt\"))\n",
    "\n",
    "\n",
    "def compute_top_k(logits, labels, k, reduction=\"mean\"):\n",
    "    _, top_ks = th.topk(logits, k, dim=-1)\n",
    "    if reduction == \"mean\":\n",
    "        return (top_ks == labels[:, None]).float().sum(dim=-1).mean().item()\n",
    "    elif reduction == \"none\":\n",
    "        return (top_ks == labels[:, None]).float().sum(dim=-1)\n",
    "\n",
    "\n",
    "def split_microbatches(microbatch, *args):\n",
    "    bs = len(args[0])\n",
    "    if microbatch == -1 or microbatch >= bs:\n",
    "        yield tuple(args)\n",
    "    else:\n",
    "        for i in range(0, bs, microbatch):\n",
    "            yield tuple(x[i : i + microbatch] if x is not None else None for x in args)\n",
    "\n",
    "\n",
    "def create_argparser():\n",
    "    defaults = dict(\n",
    "        data_dir=\"\",\n",
    "        val_data_dir=\"\",\n",
    "        noised=True,\n",
    "        iterations=150000,\n",
    "        lr=3e-4,\n",
    "        weight_decay=0.0,\n",
    "        anneal_lr=False,\n",
    "        batch_size=4,\n",
    "        microbatch=-1,\n",
    "        schedule_sampler=\"uniform\",\n",
    "        resume_checkpoint=\"\",\n",
    "        log_interval=10,\n",
    "        eval_interval=5,\n",
    "        save_interval=10000,\n",
    "    )\n",
    "    defaults.update(classifier_and_diffusion_defaults())\n",
    "    parser = argparse.ArgumentParser()\n",
    "    add_dict_to_argparser(parser, defaults)\n",
    "    return parser\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}