Fixing multiple unit tests

Merge branch 'master' of github.com:jccaicedo/DeepProfiler

deepprofiler/__main__.py

@@ -22,7 +22,7 @@
               type=click.Path("r"))
 @click.option("--config", default=None,
               help="Path to existing config file",
-              type=click.Path("r"))
+              type=click.STRING)
 @click.option("--cores", default=0,
               help="Number of CPU cores for parallel processing (all=0)",
               type=click.INT)
@@ -52,13 +52,8 @@ def cli(context, root, config, exp, cores, gpu):
         "summaries": root+"/outputs/" + exp + "/summaries/",
         "features": root+"/outputs/" + exp + "/features/"
     }
-    if config is not None:
-        context.obj["config"] = {}
-        context.obj["config"]["paths"] = {}
-        context.obj["config"]["paths"]["config"] = config
-        dirs["config"] = os.path.dirname(os.path.abspath(config))
-    else:
-        config = dirs["config"] + "/config.json"
+
+    config = dirs["config"] + "/" + config
     context.obj["cores"] = cores
     context.obj["gpu"] = gpu
 

deepprofiler/learning/profiling.py

@@ -11,7 +11,7 @@
 
 class Profile(object):
 
-    def __init__(self, config, dset):
+    def __init__(self, config, dset, gpu):
         self.config = config
         self.dset = dset
         self.num_channels = len(self.config["dataset"]["images"]["channels"])
@@ -24,7 +24,6 @@ def __init__(self, config, dset):
             .ModelClass(config, dset, gpu, self.crop_generator, self.profile_crop_generator)
         self.profile_crop_generator = self.profile_crop_generator(config, dset)
 
-
     def configure(self):        
         # Main session configuration
         configuration = tf.ConfigProto()
@@ -45,7 +44,6 @@ def configure(self):
             self.config["profile"]["feature_layer"]).output)
         self.feat_extractor.summary()
 
-
     def check(self, meta):
         output_folder = self.config["paths"]["features"]
         output_file = self.config["paths"]["features"] + "/{}_{}_{}.npz"

plugins/crop_generators/repeat_channel_crop_generator.py

@@ -1,13 +1,36 @@
 import numpy as np
-
+import tensorflow as tf
 import deepprofiler.imaging.cropping
-
+from keras.applications import inception_resnet_v2
 
 def repeat_channels(crops, labels, repeats):
+    crops = crops[np.newaxis, :]
+    labels = labels[np.newaxis, :]
+    print('crops begin', crops.shape)
+    print('crops begin', labels.shape)
+    print(repeats)
     crops = np.reshape(crops, (crops.shape[0] * crops.shape[3], crops.shape[1], crops.shape[2], 1))
+    print(crops.shape)
     crops = np.tile(crops, (1, 1, 1, repeats))
+    print(crops.shape, crops.__class__)
+    print(labels.shape)
     labels = np.tile(labels, (repeats, 1))
+    print(labels.shape)
+    #crops = tf.image.resize_images(crops, size=(299, 299))
+    print(crops.shape, crops.__class__)
     return crops, labels
+    #return inception_resnet_v2.preprocess_input(crops), labels
+    #def crop_transform(crop_ph, image_size):
+    #crops_shape = crops.shape
+    #resized_crops = tf.image.resize_images(crops, size=(299, 299))
+    #print(resized_crops.shape)
+    #reordered_channels = tf.transpose(resized_crops, [3, 0, 1, 2])
+    #print(reordered_channels.shape)
+    #reshaped_data = tf.reshape(reordered_channels, shape=[-1, 299, 299, 1])
+    #print(reshaped_data.shape)
+    #rgb_data = tf.image.grayscale_to_rgb(reordered_channels)
+    #print(rgb_data.shape)
+    #return inception_resnet_v2.preprocess_input(rgb_data), labels
 
 
 class GeneratorClass(deepprofiler.imaging.cropping.CropGenerator):
@@ -21,6 +44,7 @@ def generate(self, sess, global_step=0):
             crops = data[0]
             labels = data[1]  # TODO: enable multiple targets
             crops, labels = repeat_channels(crops, labels, self.config["dataset"]["images"]["channel_repeats"])
+            crops = crops.eval(session=sess)
             global_step += 1
             yield (crops, labels)
 
@@ -30,5 +54,22 @@ class SingleImageGeneratorClass(deepprofiler.imaging.cropping.SingleImageCropGen
     def generate(self, session, global_step=0):
         crops = self.image_pool
         labels = self.label_pool
-        crops, labels = repeat_channels(crops, labels, self.config["dataset"]["images"]["channel_repeats"])
-        yield [crops, labels]
+        #new_crops = np.array([[[[]]]])
+        #new_labels = np.array([[[[]]]])
+        for i in range(crops.shape[0]):
+            crop, label = repeat_channels(crops[i, :, :, :], labels[i, :], self.config["dataset"]["images"]["channel_repeats"])
+            #crop = crop.eval(session=session)
+            crop = session.run(tf.image.resize_images(crop, size=(299, 299)))
+            if i == 0:
+                new_crops = crop
+                new_labels = label
+            else:
+                new_crops = np.concatenate((new_crops, crop), axis=0)
+                new_labels = np.concatenate((new_labels, label), axis=0)
+
+        #print('last shape', crop.shape, label.shape)
+        #print('generator shapes', new_crops.shape, new_labels.shape)
+        #new_crops = new_crops.eval(session=session)
+        #crops, labels = repeat_channels(crops, labels, self.config["dataset"]["images"]["channel_repeats"])
+        #crops = crops.eval(session=session)
+        yield [new_crops, new_labels]

plugins/models/autoencoder.py

@@ -66,6 +66,6 @@ def define_model(config, dset):
 
 
 class ModelClass(DeepProfilerModel):
-    def __init__(self, config, dset, generator, val_generator):
-        super(ModelClass, self).__init__(config, dset, generator, val_generator)
+    def __init__(self, config, dset, gpu, generator, val_generator):
+        super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator)
         self.feature_model, self.encoder, self.decoder, self.optimizer, self.loss = define_model(config, dset)

plugins/models/cnn.py

@@ -52,6 +52,6 @@ def define_model(config, dset):
 
 
 class ModelClass(DeepProfilerModel):
-    def __init__(self, config, dset, generator, val_generator):
-        super(ModelClass, self).__init__(config, dset, generator, val_generator)
+    def __init__(self, config, dset, gpu, generator, val_generator):
+        super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator)
         self.feature_model, self.optimizer, self.loss = define_model(config, dset)

plugins/models/densenet.py

@@ -47,7 +47,7 @@ def define_model(config, dset):
     return model, optimizer, loss_func
 
 class ModelClass(DeepProfilerModel):
-    def __init__(self, config, dset, generator, val_generator):
-        super(ModelClass, self).__init__(config, dset, generator, val_generator)
+    def __init__(self, config, dset, gpu, generator, val_generator):
+        super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator)
         self.feature_model, self.optimizer, self.loss = define_model(config, dset)
 

plugins/models/gan.py

@@ -140,8 +140,8 @@ def train(self, epochs, steps_per_epoch, init_epoch):
 
 
 class ModelClass(DeepProfilerModel):
-    def __init__(self, config, dset, generator, val_generator):
-        super(ModelClass, self).__init__(config, dset, generator, val_generator)
+    def __init__(self, config, dset, gpu, generator, val_generator):
+        super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator)
         self.gan = GAN(config, self.train_crop_generator, self.val_crop_generator)
         self.feature_model = self.gan.discriminator
 

plugins/models/vae.py

@@ -88,6 +88,6 @@ def vae_loss(x, x_decoded_mean):
 
 
 class ModelClass(DeepProfilerModel):
-    def __init__(self, config, dset, generator, val_generator):
-        super(ModelClass, self).__init__(config, dset, generator, val_generator)
+    def __init__(self, config, dset, gpu, generator, val_generator):
+        super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator)
         self.feature_model, self.encoder, self.generator, self.optimizer, self.loss = define_model(config, dset)

tests/deepprofiler/learning/test_model.py

@@ -111,19 +111,22 @@ def val_crop_generator(config):
     generator = module.SingleImageGeneratorClass
     return generator
 
+@pytest.fixture(scope="function")
+def gpu():
+    return '0'
 
 @pytest.fixture(scope="function")
-def model(config, dataset, crop_generator, val_crop_generator):
+def model(config, dataset, gpu, crop_generator, val_crop_generator):
     def create():
         module = importlib.import_module("plugins.models.{}".format(config["train"]["model"]["name"]))
         importlib.invalidate_caches()
-        dpmodel = module.ModelClass(config, dataset, crop_generator, val_crop_generator)
+        dpmodel = module.ModelClass(config, dataset, gpu, crop_generator, val_crop_generator)
         return dpmodel
     return create
 
 
-def test_init(config, dataset, crop_generator, val_crop_generator):
-    dpmodel = DeepProfilerModel(config, dataset, crop_generator, val_crop_generator)
+def test_init(config, dataset, gpu, crop_generator, val_crop_generator):
+    dpmodel = DeepProfilerModel(config, dataset, gpu, crop_generator, val_crop_generator)
     assert dpmodel.feature_model is None
     assert dpmodel.config == config
     assert dpmodel.dset == dataset

tests/deepprofiler/learning/test_profiling.py

@@ -98,6 +98,12 @@ def locations(out_dir, metadata, config, make_struct):
         })
         locs.to_csv(path, index=False)
 
+
+@pytest.fixture(scope="function")
+def gpu():
+    return '0'
+
+
 @pytest.fixture(scope="function")
 def checkpoint(config, dataset):
     crop_generator = importlib.import_module(
@@ -107,20 +113,20 @@ def checkpoint(config, dataset):
         "plugins.crop_generators.{}".format(config["train"]["model"]["crop_generator"])) \
         .SingleImageGeneratorClass
     dpmodel = importlib.import_module("plugins.models.{}".format(config["train"]["model"]["name"])) \
-        .ModelClass(config, dataset, crop_generator, profile_crop_generator)
+        .ModelClass(config, dataset, gpu, crop_generator, profile_crop_generator)
     dpmodel.feature_model.compile(dpmodel.optimizer, dpmodel.loss)
     filename = os.path.join(config["paths"]["checkpoints"], config["profile"]["checkpoint"])
     dpmodel.feature_model.save_weights(filename)
     return filename
 
 
 @pytest.fixture(scope="function")
-def profile(config, dataset):
-    return deepprofiler.learning.profiling.Profile(config, dataset)
+def profile(config, dataset, gpu):
+    return deepprofiler.learning.profiling.Profile(config, dataset, gpu)
 
 
-def test_init(config, dataset):
-    prof = deepprofiler.learning.profiling.Profile(config, dataset)
+def test_init(config, dataset, gpu):
+    prof = deepprofiler.learning.profiling.Profile(config, dataset, gpu)
     test_num_channels = len(config["dataset"]["images"]["channels"])
     assert prof.config == config
     assert prof.dset == dataset
@@ -152,8 +158,8 @@ def test_extract_features(profile, metadata, locations, checkpoint):
     assert os.path.isfile(output_file)
 
 
-def test_profile(config, dataset, data, locations, checkpoint):
-    deepprofiler.learning.profiling.profile(config, dataset)
+def test_profile(config, dataset, data, locations, checkpoint, gpu):
+    deepprofiler.learning.profiling.profile(config, dataset, gpu)
     for index, row in dataset.meta.data.iterrows():
         output_file = config["paths"]["features"] + "/{}_{}_{}.npz" \
             .format(row["Metadata_Plate"], row["Metadata_Well"], row["Metadata_Site"])

tests/deepprofiler/learning/test_training.py

@@ -98,13 +98,14 @@ def locations(out_dir, metadata, config, make_struct):
 def test_learn_model(config, dataset, data, locations, out_dir, make_struct):
     epoch = 1
     verbose = 1
-    deepprofiler.learning.training.learn_model(config, dataset, epoch, verbose=verbose)
+    gpu = '0'
+    deepprofiler.learning.training.learn_model(config, dataset, gpu, epoch, verbose=verbose)
     assert os.path.exists(os.path.join(config["paths"]["checkpoints"], "checkpoint_0001.hdf5"))
     assert os.path.exists(os.path.join(config["paths"]["checkpoints"], "checkpoint_0002.hdf5"))
     assert os.path.exists(os.path.join(config["paths"]["logs"], "log.csv"))
     epoch = 3
     config["train"]["model"]["epochs"] = 4
-    deepprofiler.learning.training.learn_model(config, dataset, epoch, verbose=verbose)
+    deepprofiler.learning.training.learn_model(config, dataset, gpu, epoch, verbose=verbose)
     assert os.path.exists(os.path.join(config["paths"]["checkpoints"], "checkpoint_0003.hdf5"))
     assert os.path.exists(os.path.join(config["paths"]["checkpoints"], "checkpoint_0004.hdf5"))
     assert os.path.exists(os.path.join(config["paths"]["logs"], "log.csv"))

tests/plugins/models/test_autoencoder.py

@@ -82,6 +82,10 @@ def val_generator():
     return deepprofiler.imaging.cropping.SingleImageCropGenerator
 
 
+@pytest.fixture(scope="function")
+def gpu():
+    return '0'
+
 def test_define_model(config, dataset):
     autoencoder, encoder, decoder, optimizer, loss = plugins.models.autoencoder.define_model(config, dataset)
     assert isinstance(autoencoder, keras.Model)
@@ -91,8 +95,8 @@ def test_define_model(config, dataset):
     assert isinstance(loss, str) or callable(loss)
 
 
-def test_init(config, dataset, generator, val_generator):
-    dpmodel = plugins.models.autoencoder.ModelClass(config, dataset, generator, val_generator)
+def test_init(config, dataset, gpu, generator, val_generator):
+    dpmodel = plugins.models.autoencoder.ModelClass(config, dataset, gpu, generator, val_generator)
     autoencoder, encoder, decoder, optimizer, loss = plugins.models.autoencoder.define_model(config, dataset)
     assert dpmodel.feature_model.__eq__(autoencoder)
     assert dpmodel.encoder.__eq__(encoder)

tests/plugins/models/test_cnn.py

@@ -82,15 +82,20 @@ def val_generator():
     return deepprofiler.imaging.cropping.SingleImageCropGenerator
 
 
+@pytest.fixture(scope="function")
+def gpu():
+    return '0'
+
+
 def test_define_model(config, dataset):
     model, optimizer, loss = plugins.models.cnn.define_model(config, dataset)
     assert isinstance(model, keras.Model)
     assert isinstance(optimizer, str) or isinstance(optimizer, keras.optimizers.Optimizer)
     assert isinstance(loss, str) or callable(loss)
 
 
-def test_init(config, dataset, generator, val_generator):
-    dpmodel = plugins.models.cnn.ModelClass(config, dataset, generator, val_generator)
+def test_init(config, dataset, gpu, generator, val_generator):
+    dpmodel = plugins.models.cnn.ModelClass(config, dataset, gpu, generator, val_generator)
     model, optimizer, loss = plugins.models.cnn.define_model(config, dataset)
     assert dpmodel.feature_model.__eq__(model)
     assert dpmodel.optimizer.__eq__(optimizer)

tests/plugins/models/test_gan.py

@@ -108,9 +108,14 @@ def val_generator():
 
 
 @pytest.fixture(scope="function")
-def model(config, dataset, generator, val_generator):
+def gpu():
+    return '0'
+
+
+@pytest.fixture(scope="function")
+def model(config, dataset, gpu, generator, val_generator):
     def create():
-        return plugins.models.gan.ModelClass(config, dataset, generator, val_generator)
+        return plugins.models.gan.ModelClass(config, dataset, gpu, generator, val_generator)
     return create
 
 
@@ -140,8 +145,8 @@ def test_gan(config, generator, val_generator):
     assert not gan.discriminator_fixed.trainable
 
 
-def test_init(config, dataset, generator, val_generator):
-    dpmodel = plugins.models.gan.ModelClass(config, dataset, generator, val_generator)
+def test_init(config, dataset, gpu, generator, val_generator):
+    dpmodel = plugins.models.gan.ModelClass(config, dataset, gpu, generator, val_generator)
     gan = plugins.models.gan.GAN(config, generator, val_generator)
     assert dpmodel.gan.__eq__(gan)
     assert isinstance(dpmodel.feature_model, keras.Model)

tests/plugins/models/test_resnet.py

@@ -82,6 +82,11 @@ def val_generator():
     return deepprofiler.imaging.cropping.SingleImageCropGenerator
 
 
+@pytest.fixture(scope="function")
+def gpu():
+    return '0'
+
+
 def test_define_model(config, dataset):
     config["train"]["model"]["name"] = "resnet" 
     config["train"]["model"]["params"]["conv_blocks"] = 18
@@ -91,10 +96,10 @@ def test_define_model(config, dataset):
     assert isinstance(loss, str) or callable(loss)
 
 
-def test_init(config, dataset, generator, val_generator):
+def test_init(config, dataset, gpu, generator, val_generator):
     config["train"]["model"]["name"] = "resnet" 
     config["train"]["model"]["params"]["conv_blocks"] = 18
-    dpmodel = plugins.models.resnet.ModelClass(config, dataset, generator, val_generator)
+    dpmodel = plugins.models.resnet.ModelClass(config, dataset, gpu, generator, val_generator)
     model, optimizer, loss = plugins.models.resnet.define_model(config, dataset)
     assert dpmodel.feature_model.__eq__(model)
     assert dpmodel.optimizer.__eq__(optimizer)

tests/plugins/models/test_vae.py

@@ -82,6 +82,11 @@ def val_generator():
     return deepprofiler.imaging.cropping.SingleImageCropGenerator
 
 
+@pytest.fixture(scope="function")
+def gpu():
+    return '0'
+
+
 def test_define_model(config, dataset):
     vae, encoder, decoder, optimizer, loss = plugins.models.vae.define_model(config, dataset)
     assert isinstance(vae, keras.Model)
@@ -91,8 +96,8 @@ def test_define_model(config, dataset):
     assert isinstance(loss, str) or callable(loss)
 
 
-def test_init(config, dataset, generator, val_generator):
-    dpmodel = plugins.models.vae.ModelClass(config, dataset, generator, val_generator)
+def test_init(config, dataset, gpu, generator, val_generator):
+    dpmodel = plugins.models.vae.ModelClass(config, dataset, gpu, generator, val_generator)
     vae, encoder, decoder, optimizer, loss = plugins.models.vae.define_model(config, dataset)
     assert dpmodel.feature_model.__eq__(vae)
     assert dpmodel.encoder.__eq__(encoder)