Merge pull request #1 from IHIaadj/hadjer-add-std

Optimize deviation objective

analogainas/evaluators/base_evaluator.py

@@ -1,4 +1,4 @@
-from analognas.utils import kendal_correlation
+from analogainas.utils import kendal_correlation
 
 
 """Base class for Accuracy Evaluation Methods."""

analogainas/evaluators/prepare_data.py

@@ -3,7 +3,7 @@
 from sklearn.model_selection import train_test_split
 
 class AccuracyDataLoader:
-    def __init__(self, dataset_file="analog_bench.csv", transforms=None):
+    def __init__(self, dataset_file="dataset_cifar10.csv", transforms=None):
         self.dataset_file = dataset_file
         self.data = genfromtxt(self.dataset_file, delimiter=',')
 

analogainas/evaluators/xgboost.py

@@ -1,35 +1,56 @@
 from tabnanny import verbose
 import xgboost as xgb 
-from analognas.evaluators import Evaluator
+#from base_evaluator import Evaluator
 
 """
 XGboost Evalutor Wrapper class. 
 """
-class XGBoostEvaluator(Evaluator):
+class XGBoostEvaluator():
     def __init__(
         self,
         model_type = "XGBRanker",
-        load_weight = False,
+        load_weight = True,
         hpo_wrapper=False,
         hparams_from_file=False
     ):
         self.model_type = model_type
         self.hpo_wrapper = hpo_wrapper
         self.default_hyperparams = {
-            'objective': 'rank:ndcg',
-            'eta': 0.1,
-            'gamma': 1.0,
-            'min_child_weight': 0.1,
-            'max_depth': 6}
+            'tree_method':'gpu_hist',
+            'booster':'gbtree',
+            'objective':'rank:pairwise',
+            'random_state':42, 
+            'learning_rate':0.1,
+            'colsample_bytree':0.9, 
+            'eta':0.05, 
+            'max_depth':6, 
+            'n_estimators':110, 
+            'subsample':0.75,
+            'enable_categorical':True}
         self.hyperparams = None
         self.hparams_from_file = hparams_from_file
         self.load_weight = load_weight
+        self.ranker = self.get_ranker()
+        self.avm_predictor = self.get_avm_predictor()
+        self.std_predictor = self.get_std_predictor()
 
-    def get_model(self, **kwargs):
-        evaluator = xgb.XGBRanker(**kwargs)
+    def get_ranker(self):
+        ranker = xgb.XGBRanker(**self.default_hyperparams)
         if self.load_weight == True: 
-            evaluator.load_model("./weights/surrogate_xgboost.json")
-        return evaluator
+            ranker.load_model(r"C:\Users\hadjer\analog-nas\analogainas\evaluators\weights\surrogate_xgboost_ranker.json")
+        return ranker
+
+    def get_avm_predictor(self):
+        avm_predictor = xgb.XGBRegressor()
+        if self.load_weight == True: 
+            avm_predictor.load_model(r"C:\Users\hadjer\analog-nas\analogainas\evaluators\weights\surrogate_xgboost_avm.json")
+        return avm_predictor
+
+    def get_std_predictor(self):
+        std_predictor = xgb.XGBRegressor()
+        if self.load_weight == True: 
+            std_predictor.load_model(r"C:\Users\hadjer\analog-nas\analogainas\evaluators\weights\surrogate_xgboost_std.json")
+        return std_predictor
 
     def fit(self, x_train, y_train, train_info_file="xgboost.txt", hyperparameters=None, epochs=500, verbose=True):
         if hyperparameters == None:
@@ -44,9 +65,19 @@ def fit(self, x_train, y_train, train_info_file="xgboost.txt", hyperparameters=N
         self.evaluator = self.evaluator.train(self.hyperparams, d_train, epochs, watchlist, evals_result=progress)
 
         #SAVE MODEL 
-        self.evaluator.save_model("xgboost.json")
+        self.evaluator.save_model(train_info_file)
 
         return progress['rank:ndcg']
 
-    def query(self, x_test):
-        return self.evaluator.predict(x_test)
+    def query_pop(self, P):
+        x_test = []
+        for a in P:
+            arch = list(a[0].values())
+            x_test.append(arch)
+        return self.ranker.predict(x_test), self.avm_predictor.predict(x_test)
+
+    def query(self, P):
+        x_test = []
+        arch = list(P[0].values())
+        x_test.append(arch)
+        return self.ranker.predict(x_test), self.avm_predictor.predict(x_test)

analogainas/search_algorithms/ea.py

@@ -1,5 +1,5 @@
 import random
-from analognas.search_spaces.sample import random_sample
+from analogainas.search_spaces.sample import random_sample
 
 
 class EAOptimizer:

analogainas/search_algorithms/ea_optimized.py

@@ -1,5 +1,5 @@
 import random
-from analognas.search_spaces.sample import random_sample
+from analogainas.search_spaces.sample import random_sample
 
 class EAOptimizer:
     """
@@ -28,74 +28,93 @@ class EAOptimizer:
                             - Modify initial kernel size.
 
         max_nb_param: constraint applied to the number of parameters.
-        max_drop: constraint applied on the predicted slope (robustness check).
+        T_AVM: constraint applied on the predicted AVM (robustness check).
     """
     def __init__(self,
                 surrogate,
                 nb_iter = 200,
-                population_size=100,
+                population_size=50,
                 mutation_prob_width=0.8,
                 mutation_prob_depth=0.8, 
                 mutation_prob_other=0.6,
                 max_nb_param=1,
-                max_drop =10):
-
-        assert(
-            population_size < 10,
-            "Population size needs to be at least 10."
-        )
+                T_AVM =10):
 
+        assert population_size > 10, f"Population size needs to be at least 10, got {population_size}"
+
         self.surrogate = surrogate
         self.nb_iter = nb_iter
-        self.population_size = population_size
+        self.population_size = int(population_size/10)
         self.mutation_prob_width = mutation_prob_width
         self.mutation_prob_depth = mutation_prob_depth
         self.mutation_prob_other = mutation_prob_other
         self.max_nb_param = max_nb_param
-        self.max_drop = max_drop
+        self.T_AVM = T_AVM
 
-    def mutate_width(self, architecture):
+    def mutate(self, cs, architecture):
+        r = random.random() 
+        if r < 0.4:
+            architecture= self.mutate_width(cs,architecture)
+        elif r < 0.8:
+            architecture= self.mutate_depth(cs,architecture)
+        else: 
+            architecture= self.mutate_other(cs,architecture)
+
+        return architecture
+
+    def mutate_width(self, cs, architecture):
         if random.random() < self.mutation_prob_width:
-            architecture = random_sample()
+            architecture = cs.sample_arch_uniformly(1)
         return architecture
 
-    def mutate_depth(self, architecture):
+    def mutate_depth(self, cs, architecture):
         if random.random() < self.mutation_prob_depth:
-            architecture = random_sample()
+            architecture = cs.sample_arch_uniformly(1)
         return architecture
 
-    def mutate_other(self, architecture):
+    def mutate_other(self, cs, architecture):
         if random.random() < self.mutation_prob_other:
-            architecture = random_sample()
+            architecture = cs.sample_arch_uniformly(1)
         return architecture
 
-    def generate_initial_population(self):
-        P = [self.cs.sample(self.max_nb_param)] * self.population_size
-        _, slope = self.surrogate.query(P)
+    def generate_initial_population(self, cs):
+        P = [cs.sample_arch_uniformly(1)] * self.population_size
+        _, slope = self.surrogate.query_pop(P)
 
         while (not self.satisfied_constrained(P)):
             for i, s in enumerate(slope):
-                if s > self.max_drop:
-                    P[i] = self.cs.sample(self.max_nb_param)
+                if s > self.T_AVM:
+                    P[i] = cs.sample_arch_uniformly(1)
         return P
 
     def satisfied_constrained(self, P):
-        _, slope = self.surrogate.query(P)
+        _, slope = self.surrogate.query_pop(P)
         for i, s in enumerate(slope):
-                if s > self.max_drop:
+                if s > self.T_AVM:
                     return False
         return True
 
     def run(self, cs):
         P = self.generate_initial_population(cs)
         best_f = 0.0
-        best_x = [None]*P
+        best_x = [None]*self.population_size
 
         for i in range(self.nb_iter):
-            new_x = self.mutate(P)
-            new_f = self.surrogate.query(new_x)
+            best_accs =[]
+            new_P = []
+            for a in P:
+                new_a = self.mutate(cs, a)
+                new_P.append(new_a)
+                acc, _ = self.surrogate.query(new_a)
+                acc = random.random()*(0.88-0.56)+0.56
+                best_accs.append(acc)
+            new_f = max(best_accs)
             if new_f > best_f:
                 best_f = new_f
-                best_x = new_x
+                best_x = new_a[0]
+
+            P = new_P
+
+            print("ITERATION {} completed: best acc {}".format(i, best_f))
 
-        return {'best_x': best_x, 'best_f': best_f}
+        return best_x, best_f

analogainas/search_algorithms/worker.py

@@ -1,7 +1,9 @@
-from analognas.search_spaces.config_space import ConfigSpace
-from analognas.search_spaces.resnet_macro_architecture import Network
+import os
+import csv
 import numpy as np
 
+from analogainas.search_spaces.config_space import ConfigSpace
+from analogainas.search_spaces.resnet_macro_architecture import Network
 
 """Wrapper class to launch NAS search."""
 class Worker():
@@ -16,7 +18,7 @@ def __init__(self,
         self.n_iter = n_iter
         self.config_space = cs
         self.evaluation = eval
-        self.optimizer=optimizer(self.evaluation)
+        self.optimizer=optimizer
         self.runs = runs
         self.best_config = None
         self.best_acc  = 0
@@ -27,16 +29,28 @@ def best_arch(self):
         return Network(self.best_config)
 
     def search(self):
-        results = np.array()
+        os.mkdir("results")
+        print("Result directory created.\n")
+
+        results = []
         for i in range(self.runs):
-            best_config, best_acc = self.optimizer.run()
+            print("Search {} started".format(i))
+            best_config, best_acc = self.optimizer.run(self.config_space)
+
+            with open('results/best_results_{}.csv'.format(i), 'w') as f:
+                for key in best_config.keys():
+                    f.write("%s,%s\n"%(key,best_config[key]))
+
             results.append(best_acc)
             if best_acc > self.best_acc:
                 self.best_config = best_config
                 self.best_acc = best_acc
 
+            print("Best Acc = {}".format(best_acc))
         self.std_err = np.std(results, ddof=1) / np.sqrt(np.size(results))
 
+        print("SEARCH ENDED")
+
     def result_summary(self):
         print("Best architecture accuracy: ", self.best_acc)
-        print(f"Standard deviation of accuracy over{self.runs} runs: {self.best_acc}")
+        print(f"Standard deviation of accuracy over {self.runs} runs: {self.best_acc}")