better COCO benchmark setup flow

cuguilke · cuguilke · commit fe9cc1cbd956 · 2022-04-27T22:04:07.000+03:00
diff --git a/README.md b/README.md
@@ -33,30 +33,30 @@ tqdm~=4.58.0
 pycocotools~=2.0.0
 ```
 
-- We also include a YAML script `./acvc-pytorch.yml` that is prepared for an easy Anaconda environment setup. 
+- We also include a YAML script `acvc-pytorch.yml` that is prepared for an easy Anaconda environment setup. 
 
-- One can also use the `./requirements.txt` if one knows one's craft.
+- One can also use the `requirements.txt` if one knows one's craft.
 
 ## Training
 
-Training is done via `./run.py`. To get the up-to-date list of commands:
+Training is done via `run.py`. To get the up-to-date list of commands:
 ```shell
 python run.py --help
 ```
 
-We include a sample script `./run_experiments.sh` for a quick start.
+We include a sample script `run_experiments.sh` for a quick start.
 
 ## Analysis
 
-The benchmark results are prepared by `./analysis/GeneralizationExpProcessor.py`, which outputs LaTeX tables of the cumulative results in a .tex file.
+The benchmark results are prepared by `analysis/GeneralizationExpProcessor.py`, which outputs LaTeX tables of the cumulative results in a .tex file.
 
 - For example:
-```
+```shell
 python GeneralizationExpProcessor.py --path generalization.json --to_dir ./results --image_format pdf
 ```
 
 - You can also run distributed experiments, and merge the results later on:
-```
+```shell
 python GeneralizationExpProcessor.py --merge_logs generalization_gpu0.json generalization_gpu1.json
 ```
 
@@ -65,7 +65,22 @@ python GeneralizationExpProcessor.py --merge_logs generalization_gpu0.json gener
 COCO benchmark is especially useful for further studies on ACVC since it includes segmentation masks per image.
 
 Here are the steps to make it work:
-- TODO
+1. Download COCO 2017 [trainset](http://images.cocodataset.org/zips/train2017.zip), [valset](images.cocodataset.org/zips/val2017.zip), and [annotations](http://images.cocodataset.org/annotations/annotations_trainval2017.zip)
+2. Extract the annotations zip file into a folder named `COCO` inside your choice of `data_dir` (For example: `datasets/COCO`)
+3. Extract train and val set zip files into a subfolder named `downloads` (For example: `datasets/COCO/downloads`)
+4. Use `--first_run` argument once of while running the training script:
+```shell
+python run.py --loss CrossEntropy --epochs 1 --corruption_mode None --data_dir datasets --first_run --train_dataset COCO --test_datasets DomainNet:Real --print_config
+```
+6. If everything works fine, you will see `train2017` and `val2017` folders under `COCO`
+7. Both folders must contain 10 subfolders that belong to shared classes between COCO and DomainNet   
+8. Now, try running ACVC as well:
+```shell
+python run.py --loss CrossEntropy AttentionConsistency --epochs 1 --corruption_mode acvc --data_dir datasets --train_dataset COCO --test_datasets DomainNet:Real --print_config
+```
+9. All good? Then, you are good to go with the COCO section of `run_experiments.sh` to run multiple experiments
+10. That's it! 
+
 
 ## References
 
diff --git a/run.py b/run.py
@@ -14,7 +14,7 @@
 
 def create_config_file(config):
     # Default configurations
-    config["DEFAULT"] = {"version": "1.0.0",
+    config["DEFAULT"] = {"version": "1.0.1",
                          "model": "resnet",
                          "depth": 18,
                          "lr": 4e-3,
@@ -32,6 +32,7 @@ def create_config_file(config):
                          "test_datasets": "None",
                          "print_config": True,
                          "data_dir": "../../datasets",
+                         "first_run": False,
                          "model_dir": ".",
                          "save_model": False,
                          "knowledge_distillation": False,
@@ -63,6 +64,7 @@ def create_config_file(config):
     parser.add_argument("--test_datasets", help="list of test sets for domain generalization experiments", nargs="+")
     parser.add_argument("--to_path", help="filepath to save models with custom names", type=str)
     parser.add_argument("--data_dir", help="filepath to save datasets", type=str)
+    parser.add_argument("--first_run", help="to initiate COCO preprocessing", action="store_true")
     parser.add_argument("--model_dir", help="filepath to save models", type=str)
     parser.add_argument("--print_config", help="prints the active configurations", action="store_true")
     parser.add_argument("--save_model", help="to save the trained models", action="store_true")
@@ -104,6 +106,7 @@ def create_config_file(config):
         to_path = config["to_path"] if "to_path" in config else None
         data_dir = config["data_dir"]
         model_dir = config["model_dir"]
+        FIRST_RUN = config["first_run"]
         PRINT_CONFIG = config.getboolean("print_config")
         SAVE_MODEL = config.getboolean("save_model")
         KNOWLEDGE_DISTILLATION = config.getboolean("knowledge_distillation")
@@ -183,6 +186,7 @@ def create_config_file(config):
                                          img_mean_mode=img_mean_mode,
                                          data_dir=data_dir,
                                          distillation=KNOWLEDGE_DISTILLATION,
+                                         first_run=FIRST_RUN,
                                          wait=True)
     tester.activate() # manually trigger the dataset loader
     n_classes = tester.get_n_classes()
diff --git a/settings.ini b/settings.ini
@@ -1,5 +1,5 @@
 [DEFAULT]
-version = 1.0.0
+version = 1.0.1
 model = resnet
 depth = 18
 lr = 4e-3
@@ -18,6 +18,7 @@ test_datasets = None
 print_config = True
 save_model = False
 data_dir = ../../datasets
+first_run = False
 model_dir = .
 knowledge_distillation = False
 random_aug = False
diff --git a/testers/DomainGeneralization_tester.py b/testers/DomainGeneralization_tester.py
@@ -35,7 +35,7 @@ def step(self):
             param_group['lr'] = lr
 
 class DomainGeneralization_tester:
-    def __init__(self, train_dataset, test_dataset, img_mean_mode=None, distillation=False, wait=False, data_dir=None):
+    def __init__(self, train_dataset, test_dataset, img_mean_mode=None, distillation=False, wait=False, data_dir=None, first_run=False):
         """
         Conditional constructor to enable creating testers without immediately triggering the dataset loader.
 
@@ -47,6 +47,7 @@ def __init__(self, train_dataset, test_dataset, img_mean_mode=None, distillation
             :param wait: (bool) If True, then the constructor will onyl create the instance and
                                 wait for manual activation to actually load the dataset
             :param data_dir: (str) relative filepath to datasets
+            :param first_run: (bool) to initiate COCO benchmark preprocessing
         """
         # Base class variables
         ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -57,7 +58,8 @@ def __init__(self, train_dataset, test_dataset, img_mean_mode=None, distillation
         self.training_set_size = None
         self.img_mean_mode = img_mean_mode
         self.distillation = distillation
-        self.data_dir = os.path.join(ROOT_DIR, "../../datasets") if data_dir is None else data_dir
+        self.data_dir = os.path.join(ROOT_DIR, "../../datasets") if data_dir is None else "../%s" % data_dir
+        self.first_run = first_run
 
         # Loss func init
         self.classification_loss = None
@@ -77,7 +79,7 @@ def activate(self):
 
         # Deploy the dataset
         if "COCO" == self.train_dataset:
-            self.dataset = Datasets.load_COCO(first_run=False, train=True, img_mean_mode=self.img_mean_mode, distillation=self.distillation, data_dir=self.data_dir)
+            self.dataset = Datasets.load_COCO(first_run=self.first_run, train=True, img_mean_mode=self.img_mean_mode, distillation=self.distillation, data_dir=self.data_dir)
             self.x_test, self.y_test = Datasets.load_COCO(first_run=False, train=False, img_mean_mode=self.img_mean_mode, data_dir=self.data_dir)
 
         elif "FullDomainNet" in self.train_dataset:
