From a06e64d786b2ca36e551e68bbcf9949cbd58bcf0 Mon Sep 17 00:00:00 2001 From: Jeewon Kim <108987773+jeewonkimm2@users.noreply.github.com> Date: Sat, 8 Jul 2023 23:42:30 +0900 Subject: [PATCH 1/5] mvtec dataset load --- dataset.py | 18 ++++++++++++++++-- main.py | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/dataset.py b/dataset.py index 5a49c37..a124c02 100644 --- a/dataset.py +++ b/dataset.py @@ -24,6 +24,8 @@ def get_dataset(name, path): return get_GTSRB(path) elif name.lower() == 'tinyimagenet': return get_tinyImageNet(path) + elif name.lower() == 'mvtec': + return get_mvtec(path) def get_ImageNet(path): raw_tr = datasets.ImageFolder(path + '/tinyImageNet/tiny-imagenet-200/train') @@ -137,6 +139,18 @@ def get_GTSRB(path): return X_tr, Y_tr, X_te, Y_te +def get_mvtec(path): + train_dir = os.path.join(path, 'mvtec/train') + test_dir = os.path.join(path, 'mvtec/test') + train_data = torchvision.datasets.ImageFolder(train_dir) + test_data = torchvision.datasets.ImageFolder(test_dir) + X_tr = np.array([np.asarray(datasets.folder.default_loader(s[0])) for s in train_data.samples]) + Y_tr = torch.from_numpy(np.array(train_data.targets)) + X_te = np.array([np.asarray(datasets.folder.default_loader(s[0])) for s in test_data.samples]) + Y_te = torch.from_numpy(np.array(test_data.targets)) + + return X_tr, Y_tr, X_te, Y_te + def get_handler(name): if name.lower() == 'mnist': @@ -149,7 +163,7 @@ def get_handler(name): return DataHandler3 elif name.lower() == 'cifar100': return DataHandler3 - elif name.lower() == 'gtsrb': + elif name.lower() == 'gtsrb' or 'mvtec': return DataHandler3 elif name.lower() == 'tinyimagenet': return DataHandler3 @@ -234,7 +248,7 @@ def get_wa_handler(name): return Wa_datahandler3 elif name.lower() == 'mnist': return Wa_datahandler1 - elif name.lower() == 'gtsrb': + elif name.lower() == 'gtsrb' or 'mvtec': return Wa_datahandler3 diff --git a/main.py b/main.py index fb581c0..71961eb 100644 --- a/main.py +++ b/main.py @@ -94,7 +94,7 @@ help='load model from memory, True or False') # automatically set -# parser.add_argument("--local_rank", type=int) +parser.add_argument("--local_rank", type=int) ########################################################################## args = parser.parse_args() @@ -113,8 +113,8 @@ # True ensures the algorithm selected by CUFA is deterministic # torch.backends.cudnn.deterministic = True # torch.set_deterministic(True) - # False ensures CUDA select the same algorithm each time the application is run - torch.backends.cudnn.benchmark = False + # # False ensures CUDA select the same algorithm each time the application is run + # torch.backends.cudnn.benchmark = False ############################# Specify the hyperparameters ####################################### @@ -199,6 +199,25 @@ 'loader_te_args':{'batch_size': 1024, 'num_workers': 8}, 'normalize':{'mean': [0.3337, 0.3064, 0.3171], 'std': [0.2672, 0.2564, 0.2629]}, }, + 'mvtec': + { + 'n_class':2, + 'channels':3, + 'size': 32, + 'transform_tr': transforms.Compose([ + transforms.Resize((224, 224)), + # transforms.RandomCrop(size = 32, padding=4), + # transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize([0.3337, 0.3064, 0.3171], [0.2672, 0.2564, 0.2629])]), + 'transform_te': transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize([0.3337, 0.3064, 0.3171], [0.2672, 0.2564, 0.2629])]), + 'loader_tr_args':{'batch_size': 256, 'num_workers': 8}, + 'loader_te_args':{'batch_size': 1024, 'num_workers': 8}, + 'normalize':{'mean': [0.3337, 0.3064, 0.3171], 'std': [0.2672, 0.2564, 0.2629]}, + }, 'tinyimagenet': { 'n_class':200, @@ -238,6 +257,13 @@ ############################################################################### def main(): + # Using 2 GPUs + ngpus = torch.cuda.device_count() + torch.cuda.set_device(args.local_rank % ngpus) + device = torch.device("cuda", args.local_rank) + + + if not os.path.isdir(args.save_path): os.makedirs(args.save_path) if not os.path.isdir(args.data_path): @@ -333,6 +359,7 @@ def main(): strategy.train(alpha=alpha, n_epoch=args.n_epoch) test_acc= strategy.predict(X_te, Y_te) + acc = np.zeros(NUM_ROUND+1) acc[0] = test_acc print_log('==>> Testing accuracy {}'.format(acc[0]), log) From 725b4bc63b43b15c5ceb12ef54208ccf0aede662 Mon Sep 17 00:00:00 2001 From: Jeewon Kim <108987773+jeewonkimm2@users.noreply.github.com> Date: Sun, 9 Jul 2023 15:39:23 +0900 Subject: [PATCH 2/5] code fixed --- dataset.py | 325 +---------------------------------------------------- main.py | 150 +++++-------------------- 2 files changed, 37 insertions(+), 438 deletions(-) diff --git a/dataset.py b/dataset.py index a124c02..ed78682 100644 --- a/dataset.py +++ b/dataset.py @@ -10,134 +10,9 @@ def get_dataset(name, path): - if name.lower() == 'mnist': - return get_MNIST(path) - elif name.lower() == 'fashionmnist': - return get_FashionMNIST(path) - elif name.lower() == 'svhn': - return get_SVHN(path) - elif name.lower() == 'cifar10': - return get_CIFAR10(path) - elif name.lower() == 'cifar100': - return get_CIFAR100(path) - elif name.lower() == 'gtsrb': - return get_GTSRB(path) - elif name.lower() == 'tinyimagenet': - return get_tinyImageNet(path) - elif name.lower() == 'mvtec': + if name.lower() == 'mvtec': return get_mvtec(path) -def get_ImageNet(path): - raw_tr = datasets.ImageFolder(path + '/tinyImageNet/tiny-imagenet-200/train') - imagenet_tr_path = path +'imagenet-object-localization-challenge/ILSVRC/Data/CLS-LOC/train/' - from torchvision import transforms - transform = transforms.Compose([transforms.Resize((64, 64))]) - imagenet_folder = datasets.ImageFolder(imagenet_tr_path, transform=transform) - idx_to_class = {} - for (class_num, idx) in imagenet_folder.class_to_idx.items(): - idx_to_class[idx] = class_num - X_tr,Y_tr = [], [] - item_list = imagenet_folder.imgs - for (class_num, idx) in raw_tr.class_to_idx.items(): - new_img_num = 0 - for ii, (path, target) in enumerate(item_list): - if idx_to_class[target] == class_num: - X_tr.append(np.array(imagenet_folder[ii][0])) - Y_tr.append(idx) - new_img_num += 1 - if new_img_num >= 250: - break - - return np.array(X_tr), np.array(Y_tr) - - -def get_tinyImageNet(path): - # 100000 train 10000 test - raw_tr = datasets.ImageFolder(path + '/tinyImageNet/tiny-imagenet-200/train') - raw_te = datasets.ImageFolder(path + '/tinyImageNet/tiny-imagenet-200/val') - f = open(path + '/tinyImageNet/tiny-imagenet-200/val/val_annotations.txt') - - val_dict = {} - for line in f.readlines(): - val_dict[line.split()[0]] = raw_tr.class_to_idx[line.split()[1]] - X_tr,Y_tr,X_te, Y_te = [],[],[],[] - - div_list = [len(raw_tr)*(x+1)//10 for x in range(10)] # can not load at once, memory limitation - i=0 - for count in div_list: - loop = count - i - for j in range(loop): - image,target = raw_tr[i] - X_tr.append(np.array(image)) - Y_tr.append(target) - i += 1 - - for i in range(len(raw_te)): - img, label = raw_te[i] - img_pth = raw_te.imgs[i][0].split('/')[-1] - X_te.append(np.array(img)) - Y_te.append(val_dict[img_pth]) - - return X_tr,Y_tr,X_te, Y_te - # torch.tensor(X_tr), torch.tensor(Y_tr), torch.tensor(X_te), torch.tensor(Y_te) - -def get_MNIST(path): - raw_tr = datasets.MNIST(path + '/mnist', train=True, download=True) - raw_te = datasets.MNIST(path + '/mnist', train=False, download=True) - X_tr = raw_tr.data - Y_tr = raw_tr.targets - X_te = raw_te.data - Y_te = raw_te.targets - return X_tr, Y_tr, X_te, Y_te - -def get_FashionMNIST(path): - raw_tr = datasets.FashionMNIST(path + '/fashionmnist', train=True, download=True) - raw_te = datasets.FashionMNIST(path + '/fashionmnist', train=False, download=True) - X_tr = raw_tr.data - Y_tr = raw_tr.targets - X_te = raw_te.data - Y_te = raw_te.targets - return X_tr, Y_tr, X_te, Y_te - -def get_SVHN(path): - data_tr = datasets.SVHN(path, split='train', download=True) - data_te = datasets.SVHN(path, split='test', download=True) - X_tr = data_tr.data - Y_tr = torch.from_numpy(data_tr.labels) - X_te = data_te.data - Y_te = torch.from_numpy(data_te.labels) - return X_tr, Y_tr, X_te, Y_te - -def get_CIFAR10(path): - data_tr = datasets.CIFAR10(path + '/cifar10', train=True, download=True) - data_te = datasets.CIFAR10(path + '/cifar10', train=False, download=True) - X_tr = data_tr.data - # print(np.array(X_tr[0]).shape) - Y_tr = torch.from_numpy(np.array(data_tr.targets)) - X_te = data_te.data - Y_te = torch.from_numpy(np.array(data_te.targets)) - return X_tr, Y_tr, X_te, Y_te - -def get_CIFAR100(path): - data_tr = datasets.CIFAR100(path + '/cifar100', train=True, download=True) - data_te = datasets.CIFAR100(path + '/cifar100', train=False, download=True) - X_tr = data_tr.data - Y_tr = torch.from_numpy(np.array(data_tr.targets)) - X_te = data_te.data - Y_te = torch.from_numpy(np.array(data_te.targets)) - return X_tr, Y_tr, X_te, Y_te - -def get_GTSRB(path): - train_dir = os.path.join(path, 'gtsrb/train') - test_dir = os.path.join(path, 'gtsrb/test') - train_data = torchvision.datasets.ImageFolder(train_dir) - test_data = torchvision.datasets.ImageFolder(test_dir) - X_tr = np.array([np.asarray(datasets.folder.default_loader(s[0])) for s in train_data.samples]) - Y_tr = torch.from_numpy(np.array(train_data.targets)) - X_te = np.array([np.asarray(datasets.folder.default_loader(s[0])) for s in test_data.samples]) - Y_te = torch.from_numpy(np.array(test_data.targets)) - - return X_tr, Y_tr, X_te, Y_te def get_mvtec(path): train_dir = os.path.join(path, 'mvtec/train') @@ -153,22 +28,10 @@ def get_mvtec(path): def get_handler(name): - if name.lower() == 'mnist': - return DataHandler1 - elif name.lower() == 'fashionmnist': + if name.lower() == 'mvtec': return DataHandler1 - elif name.lower() == 'svhn': - return DataHandler2 - elif name.lower() == 'cifar10': - return DataHandler3 - elif name.lower() == 'cifar100': - return DataHandler3 - elif name.lower() == 'gtsrb' or 'mvtec': - return DataHandler3 - elif name.lower() == 'tinyimagenet': - return DataHandler3 else: - return DataHandler4 + return DataHandler2 class DataHandler1(Dataset): @@ -177,39 +40,6 @@ def __init__(self, X, Y, transform=None): self.Y = Y self.transform = transform - def __getitem__(self, index): - x, y = self.X[index], self.Y[index] - if self.transform is not None: - x = x.numpy() if not isinstance(x, np.ndarray) else x - x = Image.fromarray(x, mode='L') - x = self.transform(x) - return x, y, index - - def __len__(self): - return len(self.X) - -class DataHandler2(Dataset): - def __init__(self, X, Y, transform=None): - self.X = X - self.Y = Y - self.transform = transform - - def __getitem__(self, index): - x, y = self.X[index], self.Y[index] - if self.transform is not None: - x = Image.fromarray(np.transpose(x, (1, 2, 0))) - x = self.transform(x) - return x, y, index - - def __len__(self): - return len(self.X) - -class DataHandler3(Dataset): - def __init__(self, X, Y, transform=None): - self.X = X - self.Y = Y - self.transform = transform - def __getitem__(self, index): x, y = self.X[index], self.Y[index] if self.transform is not None: @@ -220,7 +50,7 @@ def __getitem__(self, index): def __len__(self): return len(self.X) -class DataHandler4(Dataset): +class DataHandler2(Dataset): def __init__(self, X, Y, transform=None): self.X = X self.Y = Y @@ -236,153 +66,12 @@ def __len__(self): # handler for waal def get_wa_handler(name): - if name.lower() == 'fashionmnist': + if name.lower() == 'mvtec': return Wa_datahandler1 - elif name.lower() == 'svhn': - return Wa_datahandler2 - elif name.lower() == 'cifar10': - return Wa_datahandler3 - elif name.lower() == 'cifar100': - return Wa_datahandler3 - elif name.lower() == 'tinyimagenet': - return Wa_datahandler3 - elif name.lower() == 'mnist': - return Wa_datahandler1 - elif name.lower() == 'gtsrb' or 'mvtec': - return Wa_datahandler3 class Wa_datahandler1(Dataset): - def __init__(self,X_1, Y_1, X_2, Y_2, transform = None): - """ - :param X_1: covariate from the first distribution - :param Y_1: label from the first distribution - :param X_2: - :param Y_2: - :param transform: - """ - self.X1 = X_1 - self.Y1 = Y_1 - self.X2 = X_2 - self.Y2 = Y_2 - self.transform = transform - - def __len__(self): - - # returning the minimum length of two data-sets - - return max(len(self.X1),len(self.X2)) - - def __getitem__(self, index): - Len1 = len(self.Y1) - Len2 = len(self.Y2) - - # checking the index in the range or not - - if index < Len1: - x_1 = self.X1[index] - y_1 = self.Y1[index] - - else: - - # rescaling the index to the range of Len1 - re_index = index % Len1 - - x_1 = self.X1[re_index] - y_1 = self.Y1[re_index] - - # checking second datasets - if index < Len2: - - x_2 = self.X2[index] - y_2 = self.Y2[index] - - else: - # rescaling the index to the range of Len2 - re_index = index % Len2 - - x_2 = self.X2[re_index] - y_2 = self.Y2[re_index] - - if self.transform is not None: - # print (x_1) - x_1 = Image.fromarray(x_1, mode='L') - x_1 = self.transform(x_1) - - x_2 = Image.fromarray(x_2, mode='L') - x_2 = self.transform(x_2) - - return index,x_1,y_1,x_2,y_2 - - - -class Wa_datahandler2(Dataset): - - def __init__(self,X_1, Y_1, X_2, Y_2, transform = None): - """ - :param X_1: covariate from the first distribution - :param Y_1: label from the first distribution - :param X_2: - :param Y_2: - :param transform: - """ - self.X1 = X_1 - self.Y1 = Y_1 - self.X2 = X_2 - self.Y2 = Y_2 - self.transform = transform - - def __len__(self): - - # returning the minimum length of two data-sets - - return max(len(self.X1),len(self.X2)) - - def __getitem__(self, index): - Len1 = len(self.Y1) - Len2 = len(self.Y2) - - # checking the index in the range or not - - if index < Len1: - x_1 = self.X1[index] - y_1 = self.Y1[index] - - else: - - # rescaling the index to the range of Len1 - re_index = index % Len1 - - x_1 = self.X1[re_index] - y_1 = self.Y1[re_index] - - # checking second datasets - if index < Len2: - - x_2 = self.X2[index] - y_2 = self.Y2[index] - - else: - # rescaling the index to the range of Len2 - re_index = index % Len2 - - x_2 = self.X2[re_index] - y_2 = self.Y2[re_index] - - if self.transform is not None: - - x_1 = Image.fromarray(np.transpose(x_1, (1, 2, 0))) - x_1 = self.transform(x_1) - - x_2 = Image.fromarray(np.transpose(x_2, (1, 2, 0))) - x_2 = self.transform(x_2) - - return index,x_1,y_1,x_2,y_2 - - -class Wa_datahandler3(Dataset): - def __init__(self,X_1, Y_1, X_2, Y_2, transform = None): """ :param X_1: covariate from the first distribution @@ -442,6 +131,4 @@ def __getitem__(self, index): x_2 = Image.fromarray(x_2) x_2 = self.transform(x_2) - return index,x_1,y_1,x_2,y_2 - -# get_CIFAR10('./dataset') + return index,x_1,y_1,x_2,y_2 \ No newline at end of file diff --git a/main.py b/main.py index 71961eb..47f2fc0 100644 --- a/main.py +++ b/main.py @@ -13,6 +13,11 @@ import query_strategies import models from utils import print_log + +from sklearn.metrics import roc_auc_score +import sklearn.metrics as metrics + + # import torch.distributed as dist os.environ['CUBLAS_WORKSPACE_CONFIG']= ':16:8' @@ -118,87 +123,7 @@ ############################# Specify the hyperparameters ####################################### -args_pool = {'mnist': - { - 'n_class':10, - 'channels':1, - 'size': 28, - 'transform_tr': transforms.Compose([ - transforms.RandomHorizontalFlip(), - transforms.ToTensor(), - transforms.Normalize((0.1307,), (0.3081,))]), - 'transform_te': transforms.Compose([transforms.ToTensor(), - transforms.Normalize((0.1307,), (0.3081,))]), - 'loader_tr_args':{'batch_size': 128, 'num_workers': 8}, - 'loader_te_args':{'batch_size': 1024, 'num_workers': 8}, - 'normalize':{'mean': (0.1307,), 'std': (0.3081,)}, - }, - 'fashionmnist': - { - 'n_class':10, - 'channels':1, - 'size': 28, - 'transform_tr': transforms.Compose([ - transforms.RandomHorizontalFlip(), - transforms.ToTensor(), - transforms.Normalize((0.1307,), (0.3081,))]), - 'transform_te': transforms.Compose([transforms.ToTensor(), - transforms.Normalize((0.1307,), (0.3081,))]), - 'loader_tr_args':{'batch_size': 256, 'num_workers': 1}, - 'loader_te_args':{'batch_size': 1024, 'num_workers': 1}, - 'normalize':{'mean': (0.1307,), 'std': (0.3081,)}, - }, - 'svhn': - { - 'n_class':10, - 'channels':3, - 'size': 32, - 'transform_tr': transforms.Compose([ - transforms.RandomCrop(size = 32, padding=4), - transforms.RandomHorizontalFlip(), - transforms.ToTensor(), - transforms.Normalize((0.4377, 0.4438, 0.4728), (0.1980, 0.2010, 0.1970))]), - 'transform_te': transforms.Compose([transforms.ToTensor(), - transforms.Normalize((0.4377, 0.4438, 0.4728), (0.1980, 0.2010, 0.1970))]), - 'loader_tr_args':{'batch_size': 128, 'num_workers': 8}, - 'loader_te_args':{'batch_size': 1024, 'num_workers': 8}, - 'normalize':{'mean': (0.4377, 0.4438, 0.4728), 'std': (0.1980, 0.2010, 0.1970)}, - }, - 'cifar10': - { - 'n_class':10, - 'channels':3, - 'size': 32, - 'transform_tr': transforms.Compose([ - transforms.RandomCrop(size = 32, padding=4), - transforms.RandomHorizontalFlip(), - transforms.ToTensor(), - transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))]), - 'transform_te': transforms.Compose([transforms.ToTensor(), - transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))]), - 'loader_tr_args':{'batch_size': 256, 'num_workers': 8}, - 'loader_te_args':{'batch_size': 512, 'num_workers': 8}, - 'normalize':{'mean': (0.4914, 0.4822, 0.4465), 'std': (0.2470, 0.2435, 0.2616)}, - }, - 'gtsrb': - { - 'n_class':43, - 'channels':3, - 'size': 32, - 'transform_tr': transforms.Compose([ - transforms.Resize((32, 32)), - transforms.RandomCrop(size = 32, padding=4), - transforms.RandomHorizontalFlip(), - transforms.ToTensor(), - transforms.Normalize([0.3337, 0.3064, 0.3171], [0.2672, 0.2564, 0.2629])]), - 'transform_te': transforms.Compose([ - transforms.Resize((32, 32)), - transforms.ToTensor(), - transforms.Normalize([0.3337, 0.3064, 0.3171], [0.2672, 0.2564, 0.2629])]), - 'loader_tr_args':{'batch_size': 256, 'num_workers': 8}, - 'loader_te_args':{'batch_size': 1024, 'num_workers': 8}, - 'normalize':{'mean': [0.3337, 0.3064, 0.3171], 'std': [0.2672, 0.2564, 0.2629]}, - }, +args_pool = { 'mvtec': { 'n_class':2, @@ -217,39 +142,6 @@ 'loader_tr_args':{'batch_size': 256, 'num_workers': 8}, 'loader_te_args':{'batch_size': 1024, 'num_workers': 8}, 'normalize':{'mean': [0.3337, 0.3064, 0.3171], 'std': [0.2672, 0.2564, 0.2629]}, - }, - 'tinyimagenet': - { - 'n_class':200, - 'channels':3, - 'size': 64, - 'transform_tr': transforms.Compose([ - transforms.RandomCrop(size = 64, padding=4), - transforms.RandomHorizontalFlip(), - transforms.ToTensor(), - transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))]), - 'transform_te': transforms.Compose([ - transforms.ToTensor(), - transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))]), - 'loader_tr_args':{'batch_size': 1024, 'num_workers': 4}, - 'loader_te_args':{'batch_size': 512, 'num_workers': 4}, - 'normalize':{'mean': (0.485, 0.456, 0.406), 'std': (0.229, 0.224, 0.225)}, - }, - 'cifar100': - { - 'n_class':100, - 'channels':3, - 'size': 32, - 'transform_tr': transforms.Compose([ - transforms.RandomCrop(size = 32, padding=4), - transforms.RandomHorizontalFlip(), - transforms.ToTensor(), - transforms.Normalize((0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761))]), - 'transform_te': transforms.Compose([transforms.ToTensor(), - transforms.Normalize((0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761))]), - 'loader_tr_args':{'batch_size': 2048, 'num_workers': 4}, - 'loader_te_args':{'batch_size': 512, 'num_workers': 8}, - 'normalize':{'mean': (0.5071, 0.4867, 0.4408), 'std': (0.2675, 0.2565, 0.2761)}, } } @@ -313,11 +205,15 @@ def main(): if args.dataset == 'mnist': args.schedule = [20, 40] - args.nEnd = args.nEnd if args.nEnd != -1 else 100 + # args.nEnd = args.nEnd if args.nEnd != -1 else 100 + args.nEnd = args.nEnd if args.nEnd != -1 else 50 args.nQuery = args.nQuery if args.nQuery != -1 else (args.nEnd - args.nStart) - - NUM_INIT_LB = int(args.nStart*n_pool/100) - NUM_QUERY = int(args.nQuery*n_pool/100) if args.nStart!= 100 else 0 + + args.nStart = 1 + # NUM_INIT_LB = int(args.nStart*n_pool/100) + NUM_INIT_LB = 5 + # NUM_QUERY = int(args.nQuery*n_pool/100) if args.nStart!= 100 else 0 + NUM_QUERY = 1 NUM_ROUND = int((int(args.nEnd*n_pool/100) - NUM_INIT_LB)/ NUM_QUERY) if args.nStart!= 100 else 0 if NUM_QUERY != 0: if (int(args.nEnd*n_pool/100) - NUM_INIT_LB)% NUM_QUERY != 0: @@ -359,10 +255,19 @@ def main(): strategy.train(alpha=alpha, n_epoch=args.n_epoch) test_acc= strategy.predict(X_te, Y_te) + pred_proba = strategy.predict_prob(X_te, Y_te) # Predict class probabilities + # Calculate AUROC + y_true = Y_te + y_scores = pred_proba[:, 1] # Use the probabilities of the positive class + auroc_bf = metrics.roc_auc_score(y_true, y_scores) + acc = np.zeros(NUM_ROUND+1) acc[0] = test_acc + print_log('==>> Testing accuracy {}'.format(acc[0]), log) + print_log('==>> Testing auroc {}'.format(auroc_bf), log) + out_file = os.path.join(args.save_path, args.save_file) for rd in range(1, NUM_ROUND+1): @@ -381,7 +286,9 @@ def main(): # update strategy.update(idxs_lb) - best_test_acc = strategy.train(alpha=alpha, n_epoch=args.n_epoch) + best_test_acc, auroc_it = strategy.train(alpha=alpha, n_epoch=args.n_epoch) + + t_iter = time.time() - ts @@ -389,6 +296,7 @@ def main(): # test_acc = strategy.predict(X_te, Y_te) acc[rd] = best_test_acc print_log(str(sum(idxs_lb)) + '\t' + 'testing accuracy {}'.format(acc[rd]), log) + print_log(str(sum(idxs_lb)) + '\t' + 'testing auroc {}'.format(auroc_it), log) print_log("logging...", log) with open(out_file, 'a+') as f: @@ -408,6 +316,10 @@ def main(): acc[0], acc[rd], acc[rd] - acc[0], + 'aurocCompare', + auroc_bf, + auroc_it, + auroc_it - auroc_bf, 't_query', tp, 't_iter', From 65cd1963201740471de1ff1fa7bda71fcdc8407d Mon Sep 17 00:00:00 2001 From: Jeewon Kim <108987773+jeewonkimm2@users.noreply.github.com> Date: Sun, 9 Jul 2023 15:39:51 +0900 Subject: [PATCH 3/5] auroc code added --- query_strategies/strategy.py | 63 +++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/query_strategies/strategy.py b/query_strategies/strategy.py index 301dced..5aa9735 100644 --- a/query_strategies/strategy.py +++ b/query_strategies/strategy.py @@ -16,6 +16,8 @@ from tqdm import tqdm from .util import AugMixDataset from sklearn.metrics import pairwise_distances +import sklearn.metrics as metrics + class Strategy: def __init__(self, X, Y, X_te, Y_te, idxs_lb, net, handler, args): self.X = X # vector @@ -90,56 +92,51 @@ def _train(self, epoch, loader_tr, optimizer): def train(self, alpha=0.1, n_epoch=10): - self.clf = deepcopy(self.net) - # if torch.cuda.device_count() > 1: + self.clf = deepcopy(self.net) print("Let's use", torch.cuda.device_count(), "GPUs!") - # self.clf = nn.parallel.DistributedDataParallel(self.clf, - # find_unused_parameters=True, - # ) self.clf = nn.DataParallel(self.clf).to(self.device) parameters = self.clf.parameters() - optimizer = optim.SGD(parameters, lr = self.args.lr, weight_decay=5e-4, momentum=self.args.momentum) + optimizer = optim.SGD(parameters, lr=self.args.lr, weight_decay=5e-4, momentum=self.args.momentum) idxs_train = np.arange(self.n_pool)[self.idxs_lb] - epoch_time = AverageMeter() recorder = RecorderMeter(n_epoch) - epoch = 0 + epoch = 0 train_acc = 0. best_test_acc = 0. if idxs_train.shape[0] != 0: transform = self.args.transform_tr - train_data = self.handler(self.X[idxs_train], - torch.Tensor(self.Y[idxs_train]).long() if type(self.Y) is np.ndarray else torch.Tensor(self.Y.numpy()[idxs_train]).long(), + train_data = self.handler(self.X[idxs_train], + torch.Tensor(self.Y[idxs_train]).long() if type(self.Y) is np.ndarray else torch.Tensor( + self.Y.numpy()[idxs_train]).long(), transform=transform) - loader_tr = DataLoader(train_data, - shuffle=True, - pin_memory=True, - # sampler = DistributedSampler(train_data), - worker_init_fn=self.seed_worker, - generator=self.g, - **self.args.loader_tr_args) + loader_tr = DataLoader(train_data, + shuffle=True, + pin_memory=True, + worker_init_fn=self.seed_worker, + generator=self.g, + **self.args.loader_tr_args) for epoch in range(n_epoch): ts = time.time() - current_learning_rate, _ = adjust_learning_rate(optimizer, epoch, self.args.gammas, self.args.schedule, self.args) - - # Display simulation time + current_learning_rate, _ = adjust_learning_rate(optimizer, epoch, self.args.gammas, self.args.schedule, + self.args) + need_hour, need_mins, need_secs = convert_secs2time(epoch_time.avg * (n_epoch - epoch)) - need_time = '[{} Need: {:02d}:{:02d}:{:02d}]'.format(self.args.strategy, need_hour, need_mins, need_secs) - - # train one epoch + need_time = '[{} Need: {:02d}:{:02d}:{:02d}]'.format(self.args.strategy, need_hour, need_mins, + need_secs) + train_acc, train_los = self._train(epoch, loader_tr, optimizer) test_acc = self.predict(self.X_te, self.Y_te) - # measure elapsed time + epoch_time.update(time.time() - ts) print('\n==>>{:s} [Epoch={:03d}/{:03d}] {:s} [LR={:6.4f}]'.format(time_string(), epoch, n_epoch, - need_time, current_learning_rate - ) \ - + ' [Best : Test Accuracy={:.2f}, Error={:.2f}]'.format(recorder.max_accuracy(False), - 1. - recorder.max_accuracy(False))) + need_time, current_learning_rate + ) \ + + ' [Best : Test Accuracy={:.2f}, Error={:.2f}]'.format(recorder.max_accuracy(False), + 1. - recorder.max_accuracy(False))) recorder.update(epoch, train_los, train_acc, 0, test_acc) if self.args.save_model and test_acc > best_test_acc: @@ -149,7 +146,15 @@ def train(self, alpha=0.1, n_epoch=10): self.clf = self.clf.module best_test_acc = recorder.max_accuracy(istrain=False) - return best_test_acc + + # Calculate AUROC + test_probs = self.predict_prob(self.X_te, self.Y_te) + y_true = np.array(self.Y_te) + y_scores = test_probs[:, 1] # Use the probability of the positive class + auroc = metrics.roc_auc_score(y_true, y_scores) + + return best_test_acc, auroc + def predict(self, X, Y): From 50305682a028d89d05c4bde3a2b3dea1a2d1f693 Mon Sep 17 00:00:00 2001 From: so-hee bae <123538321+bae-sohee@users.noreply.github.com> Date: Mon, 10 Jul 2023 21:28:39 +0900 Subject: [PATCH 4/5] code fixed --- main.py | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/main.py b/main.py index 47f2fc0..a1aa5d0 100644 --- a/main.py +++ b/main.py @@ -99,7 +99,7 @@ help='load model from memory, True or False') # automatically set -parser.add_argument("--local_rank", type=int) +parser.add_argument("--local_rank", type=int ) ########################################################################## args = parser.parse_args() @@ -128,7 +128,7 @@ { 'n_class':2, 'channels':3, - 'size': 32, + 'size': 1024, 'transform_tr': transforms.Compose([ transforms.Resize((224, 224)), # transforms.RandomCrop(size = 32, padding=4), @@ -201,20 +201,16 @@ def main(): n_pool = len(Y_tr) n_test = len(Y_te) - # parameters - if args.dataset == 'mnist': - args.schedule = [20, 40] - # args.nEnd = args.nEnd if args.nEnd != -1 else 100 args.nEnd = args.nEnd if args.nEnd != -1 else 50 args.nQuery = args.nQuery if args.nQuery != -1 else (args.nEnd - args.nStart) - args.nStart = 1 # NUM_INIT_LB = int(args.nStart*n_pool/100) - NUM_INIT_LB = 5 + NUM_INIT_LB = 6 # NUM_QUERY = int(args.nQuery*n_pool/100) if args.nStart!= 100 else 0 NUM_QUERY = 1 - NUM_ROUND = int((int(args.nEnd*n_pool/100) - NUM_INIT_LB)/ NUM_QUERY) if args.nStart!= 100 else 0 + # NUM_ROUND = int((int(args.nEnd*n_pool/100) - NUM_INIT_LB)/ NUM_QUERY) if args.nStart!= 100 else 0 + NUM_ROUND = 100 if NUM_QUERY != 0: if (int(args.nEnd*n_pool/100) - NUM_INIT_LB)% NUM_QUERY != 0: NUM_ROUND += 1 @@ -257,7 +253,7 @@ def main(): pred_proba = strategy.predict_prob(X_te, Y_te) # Predict class probabilities # Calculate AUROC - y_true = Y_te + y_true = np.array(Y_te) y_scores = pred_proba[:, 1] # Use the probabilities of the positive class auroc_bf = metrics.roc_auc_score(y_true, y_scores) @@ -305,13 +301,9 @@ def main(): args.strategy, args.seed, 'budget', - args.nEnd, - 'nStart', - args.nStart, - 'nQuery', - args.nQuery, + NUM_ROUND*NUM_QUERY, 'labeled', - min(args.nStart + args.nQuery*rd, args.nEnd), + str(sum(idxs_lb)), 'accCompare', acc[0], acc[rd], From 4fed97ffecfac7284786798bd3bbf4cf84c78353 Mon Sep 17 00:00:00 2001 From: so-hee bae <123538321+bae-sohee@users.noreply.github.com> Date: Sun, 23 Jul 2023 16:20:54 +0900 Subject: [PATCH 5/5] code fixed macro f1 score --- main.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/main.py b/main.py index a1aa5d0..fd40ee5 100644 --- a/main.py +++ b/main.py @@ -15,6 +15,7 @@ from utils import print_log from sklearn.metrics import roc_auc_score +from sklearn.metrics import f1_score import sklearn.metrics as metrics @@ -71,7 +72,7 @@ # training hyperparameters parser.add_argument('--optimizer', type=str, - default='SGD', + default='Adam', choices=['SGD', 'Adam', 'YF']) parser.add_argument('--n_epoch', type=int, default=100, help='number of training epochs in each iteration') @@ -81,7 +82,7 @@ default=[80, 120], help='Decrease learning rate at these epochs.') parser.add_argument('--momentum', type=float, default=0.9, help='Momentum.') -parser.add_argument('--lr', type=float, default=0.1, help='learning rate. 0.01 for semi') +parser.add_argument('--lr', type=float, default=0.001, help='learning rate. 0.01 for semi') parser.add_argument('--gammas', type=float, nargs='+', @@ -154,8 +155,6 @@ def main(): torch.cuda.set_device(args.local_rank % ngpus) device = torch.device("cuda", args.local_rank) - - if not os.path.isdir(args.save_path): os.makedirs(args.save_path) if not os.path.isdir(args.data_path): @@ -252,11 +251,15 @@ def main(): test_acc= strategy.predict(X_te, Y_te) pred_proba = strategy.predict_prob(X_te, Y_te) # Predict class probabilities - # Calculate AUROC + # y_pred = np.argmax(pred_proba, axis=1) + y_true = np.array(Y_te) y_scores = pred_proba[:, 1] # Use the probabilities of the positive class + + # Calculate AUROC_bf auroc_bf = metrics.roc_auc_score(y_true, y_scores) - + # Calculate macro_f1_bf + # macro_f1_bf = f1_score(y_true, y_pred, average='macro') acc = np.zeros(NUM_ROUND+1) acc[0] = test_acc @@ -283,8 +286,10 @@ def main(): # update strategy.update(idxs_lb) best_test_acc, auroc_it = strategy.train(alpha=alpha, n_epoch=args.n_epoch) - - + + pred_proba_it = strategy.predict_prob(X_te, Y_te) + y_pred_it = np.argmax(pred_proba_it, axis=1) + macro_f1_it = f1_score(y_true, y_pred_it, average='macro') t_iter = time.time() - ts @@ -293,6 +298,7 @@ def main(): acc[rd] = best_test_acc print_log(str(sum(idxs_lb)) + '\t' + 'testing accuracy {}'.format(acc[rd]), log) print_log(str(sum(idxs_lb)) + '\t' + 'testing auroc {}'.format(auroc_it), log) + print_log(str(sum(idxs_lb)) + '\t' + 'testing macro f1 {}'.format(macro_f1_it), log) print_log("logging...", log) with open(out_file, 'a+') as f: @@ -312,6 +318,8 @@ def main(): auroc_bf, auroc_it, auroc_it - auroc_bf, + 'macro f1', + macro_f1_it, 't_query', tp, 't_iter',