From 0a4fdfe4ca22f59ee5d7c19c3b7ce078df78e1e7 Mon Sep 17 00:00:00 2001 From: Nick Date: Tue, 24 Sep 2019 18:35:36 +0200 Subject: [PATCH 1/4] Gpu param for command line, tests. --- deepprofiler/__main__.py | 14 ++++++------- plugins/models/autoencoder.py | 4 ++-- plugins/models/cnn.py | 4 ++-- plugins/models/densenet.py | 4 ++-- plugins/models/gan.py | 4 ++-- plugins/models/vae.py | 4 ++-- tests/deepprofiler/learning/test_model.py | 11 ++++++---- tests/deepprofiler/learning/test_profiling.py | 20 ++++++++++++------- tests/deepprofiler/learning/test_training.py | 5 +++-- tests/plugins/models/test_autoencoder.py | 8 ++++++-- tests/plugins/models/test_cnn.py | 9 +++++++-- tests/plugins/models/test_gan.py | 13 ++++++++---- tests/plugins/models/test_resnet.py | 9 +++++++-- tests/plugins/models/test_vae.py | 9 +++++++-- 14 files changed, 75 insertions(+), 43 deletions(-) diff --git a/deepprofiler/__main__.py b/deepprofiler/__main__.py index 3c415ea..62df75d 100644 --- a/deepprofiler/__main__.py +++ b/deepprofiler/__main__.py @@ -22,7 +22,7 @@ type=click.Path("r")) @click.option("--config", default=None, help="Path to existing config file", - type=click.Path("r")) + type=click.STRING) @click.option("--cores", default=0, help="Number of CPU cores for parallel processing (all=0)", type=click.INT) @@ -52,13 +52,11 @@ def cli(context, root, config, exp, cores, gpu): "summaries": root+"/outputs/" + exp + "/summaries/", "features": root+"/outputs/" + exp + "/features/" } - if config is not None: - context.obj["config"] = {} - context.obj["config"]["paths"] = {} - context.obj["config"]["paths"]["config"] = config - dirs["config"] = os.path.dirname(os.path.abspath(config)) - else: - config = dirs["config"] + "/config.json" + + context.obj["config"] = {} + context.obj["config"]["paths"] = {} + context.obj["config"]["paths"]["config"] = root+"/inputs/config/" + config + context.obj["cores"] = cores context.obj["gpu"] = gpu diff --git a/plugins/models/autoencoder.py b/plugins/models/autoencoder.py index a7937f3..a1bad9d 100644 --- a/plugins/models/autoencoder.py +++ b/plugins/models/autoencoder.py @@ -66,6 +66,6 @@ def define_model(config, dset): class ModelClass(DeepProfilerModel): - def __init__(self, config, dset, generator, val_generator): - super(ModelClass, self).__init__(config, dset, generator, val_generator) + def __init__(self, config, dset, gpu, generator, val_generator): + super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator) self.feature_model, self.encoder, self.decoder, self.optimizer, self.loss = define_model(config, dset) diff --git a/plugins/models/cnn.py b/plugins/models/cnn.py index 3b3245f..813f27c 100644 --- a/plugins/models/cnn.py +++ b/plugins/models/cnn.py @@ -52,6 +52,6 @@ def define_model(config, dset): class ModelClass(DeepProfilerModel): - def __init__(self, config, dset, generator, val_generator): - super(ModelClass, self).__init__(config, dset, generator, val_generator) + def __init__(self, config, dset, gpu, generator, val_generator): + super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator) self.feature_model, self.optimizer, self.loss = define_model(config, dset) diff --git a/plugins/models/densenet.py b/plugins/models/densenet.py index 188f12b..fdf5d1f 100644 --- a/plugins/models/densenet.py +++ b/plugins/models/densenet.py @@ -47,7 +47,7 @@ def define_model(config, dset): return model, optimizer, loss_func class ModelClass(DeepProfilerModel): - def __init__(self, config, dset, generator, val_generator): - super(ModelClass, self).__init__(config, dset, generator, val_generator) + def __init__(self, config, dset, gpu, generator, val_generator): + super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator) self.feature_model, self.optimizer, self.loss = define_model(config, dset) diff --git a/plugins/models/gan.py b/plugins/models/gan.py index 1ef9b8c..c8f6280 100644 --- a/plugins/models/gan.py +++ b/plugins/models/gan.py @@ -140,8 +140,8 @@ def train(self, epochs, steps_per_epoch, init_epoch): class ModelClass(DeepProfilerModel): - def __init__(self, config, dset, generator, val_generator): - super(ModelClass, self).__init__(config, dset, generator, val_generator) + def __init__(self, config, dset, gpu, generator, val_generator): + super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator) self.gan = GAN(config, self.train_crop_generator, self.val_crop_generator) self.feature_model = self.gan.discriminator diff --git a/plugins/models/vae.py b/plugins/models/vae.py index 5d4f939..f2df4aa 100644 --- a/plugins/models/vae.py +++ b/plugins/models/vae.py @@ -88,6 +88,6 @@ def vae_loss(x, x_decoded_mean): class ModelClass(DeepProfilerModel): - def __init__(self, config, dset, generator, val_generator): - super(ModelClass, self).__init__(config, dset, generator, val_generator) + def __init__(self, config, dset, gpu, generator, val_generator): + super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator) self.feature_model, self.encoder, self.generator, self.optimizer, self.loss = define_model(config, dset) diff --git a/tests/deepprofiler/learning/test_model.py b/tests/deepprofiler/learning/test_model.py index 7ce836f..b202480 100644 --- a/tests/deepprofiler/learning/test_model.py +++ b/tests/deepprofiler/learning/test_model.py @@ -111,19 +111,22 @@ def val_crop_generator(config): generator = module.SingleImageGeneratorClass return generator +@pytest.fixture(scope="function") +def gpu(): + return '0' @pytest.fixture(scope="function") -def model(config, dataset, crop_generator, val_crop_generator): +def model(config, dataset, gpu, crop_generator, val_crop_generator): def create(): module = importlib.import_module("plugins.models.{}".format(config["train"]["model"]["name"])) importlib.invalidate_caches() - dpmodel = module.ModelClass(config, dataset, crop_generator, val_crop_generator) + dpmodel = module.ModelClass(config, dataset, gpu, crop_generator, val_crop_generator) return dpmodel return create -def test_init(config, dataset, crop_generator, val_crop_generator): - dpmodel = DeepProfilerModel(config, dataset, crop_generator, val_crop_generator) +def test_init(config, dataset, gpu, crop_generator, val_crop_generator): + dpmodel = DeepProfilerModel(config, dataset, gpu, crop_generator, val_crop_generator) assert dpmodel.feature_model is None assert dpmodel.config == config assert dpmodel.dset == dataset diff --git a/tests/deepprofiler/learning/test_profiling.py b/tests/deepprofiler/learning/test_profiling.py index b9a3e9b..b0ce1c7 100644 --- a/tests/deepprofiler/learning/test_profiling.py +++ b/tests/deepprofiler/learning/test_profiling.py @@ -98,6 +98,12 @@ def locations(out_dir, metadata, config, make_struct): }) locs.to_csv(path, index=False) + +@pytest.fixture(scope="function") +def gpu(): + return '0' + + @pytest.fixture(scope="function") def checkpoint(config, dataset): crop_generator = importlib.import_module( @@ -107,7 +113,7 @@ def checkpoint(config, dataset): "plugins.crop_generators.{}".format(config["train"]["model"]["crop_generator"])) \ .SingleImageGeneratorClass dpmodel = importlib.import_module("plugins.models.{}".format(config["train"]["model"]["name"])) \ - .ModelClass(config, dataset, crop_generator, profile_crop_generator) + .ModelClass(config, dataset, gpu, crop_generator, profile_crop_generator) dpmodel.feature_model.compile(dpmodel.optimizer, dpmodel.loss) filename = os.path.join(config["paths"]["checkpoints"], config["profile"]["checkpoint"]) dpmodel.feature_model.save_weights(filename) @@ -115,12 +121,12 @@ def checkpoint(config, dataset): @pytest.fixture(scope="function") -def profile(config, dataset): - return deepprofiler.learning.profiling.Profile(config, dataset) +def profile(config, dataset, gpu): + return deepprofiler.learning.profiling.Profile(config, dataset, gpu) -def test_init(config, dataset): - prof = deepprofiler.learning.profiling.Profile(config, dataset) +def test_init(config, dataset, gpu): + prof = deepprofiler.learning.profiling.Profile(config, dataset, gpu) test_num_channels = len(config["dataset"]["images"]["channels"]) assert prof.config == config assert prof.dset == dataset @@ -152,8 +158,8 @@ def test_extract_features(profile, metadata, locations, checkpoint): assert os.path.isfile(output_file) -def test_profile(config, dataset, data, locations, checkpoint): - deepprofiler.learning.profiling.profile(config, dataset) +def test_profile(config, dataset, data, locations, checkpoint, gpu): + deepprofiler.learning.profiling.profile(config, dataset, gpu) for index, row in dataset.meta.data.iterrows(): output_file = config["paths"]["features"] + "/{}_{}_{}.npz" \ .format(row["Metadata_Plate"], row["Metadata_Well"], row["Metadata_Site"]) diff --git a/tests/deepprofiler/learning/test_training.py b/tests/deepprofiler/learning/test_training.py index e44f7e5..06165e2 100644 --- a/tests/deepprofiler/learning/test_training.py +++ b/tests/deepprofiler/learning/test_training.py @@ -98,13 +98,14 @@ def locations(out_dir, metadata, config, make_struct): def test_learn_model(config, dataset, data, locations, out_dir, make_struct): epoch = 1 verbose = 1 - deepprofiler.learning.training.learn_model(config, dataset, epoch, verbose=verbose) + gpu = '0' + deepprofiler.learning.training.learn_model(config, dataset, gpu, epoch, verbose=verbose) assert os.path.exists(os.path.join(config["paths"]["checkpoints"], "checkpoint_0001.hdf5")) assert os.path.exists(os.path.join(config["paths"]["checkpoints"], "checkpoint_0002.hdf5")) assert os.path.exists(os.path.join(config["paths"]["logs"], "log.csv")) epoch = 3 config["train"]["model"]["epochs"] = 4 - deepprofiler.learning.training.learn_model(config, dataset, epoch, verbose=verbose) + deepprofiler.learning.training.learn_model(config, dataset, gpu, epoch, verbose=verbose) assert os.path.exists(os.path.join(config["paths"]["checkpoints"], "checkpoint_0003.hdf5")) assert os.path.exists(os.path.join(config["paths"]["checkpoints"], "checkpoint_0004.hdf5")) assert os.path.exists(os.path.join(config["paths"]["logs"], "log.csv")) diff --git a/tests/plugins/models/test_autoencoder.py b/tests/plugins/models/test_autoencoder.py index 82f7273..fa0a5f0 100644 --- a/tests/plugins/models/test_autoencoder.py +++ b/tests/plugins/models/test_autoencoder.py @@ -82,6 +82,10 @@ def val_generator(): return deepprofiler.imaging.cropping.SingleImageCropGenerator +@pytest.fixture(scope="function") +def gpu(): + return '0' + def test_define_model(config, dataset): autoencoder, encoder, decoder, optimizer, loss = plugins.models.autoencoder.define_model(config, dataset) assert isinstance(autoencoder, keras.Model) @@ -91,8 +95,8 @@ def test_define_model(config, dataset): assert isinstance(loss, str) or callable(loss) -def test_init(config, dataset, generator, val_generator): - dpmodel = plugins.models.autoencoder.ModelClass(config, dataset, generator, val_generator) +def test_init(config, dataset, gpu, generator, val_generator): + dpmodel = plugins.models.autoencoder.ModelClass(config, dataset, gpu, generator, val_generator) autoencoder, encoder, decoder, optimizer, loss = plugins.models.autoencoder.define_model(config, dataset) assert dpmodel.feature_model.__eq__(autoencoder) assert dpmodel.encoder.__eq__(encoder) diff --git a/tests/plugins/models/test_cnn.py b/tests/plugins/models/test_cnn.py index 961db9b..e79e094 100644 --- a/tests/plugins/models/test_cnn.py +++ b/tests/plugins/models/test_cnn.py @@ -82,6 +82,11 @@ def val_generator(): return deepprofiler.imaging.cropping.SingleImageCropGenerator +@pytest.fixture(scope="function") +def gpu(): + return '0' + + def test_define_model(config, dataset): model, optimizer, loss = plugins.models.cnn.define_model(config, dataset) assert isinstance(model, keras.Model) @@ -89,8 +94,8 @@ def test_define_model(config, dataset): assert isinstance(loss, str) or callable(loss) -def test_init(config, dataset, generator, val_generator): - dpmodel = plugins.models.cnn.ModelClass(config, dataset, generator, val_generator) +def test_init(config, dataset, gpu, generator, val_generator): + dpmodel = plugins.models.cnn.ModelClass(config, dataset, gpu, generator, val_generator) model, optimizer, loss = plugins.models.cnn.define_model(config, dataset) assert dpmodel.feature_model.__eq__(model) assert dpmodel.optimizer.__eq__(optimizer) diff --git a/tests/plugins/models/test_gan.py b/tests/plugins/models/test_gan.py index 0a8ccd9..938c4ff 100644 --- a/tests/plugins/models/test_gan.py +++ b/tests/plugins/models/test_gan.py @@ -108,9 +108,14 @@ def val_generator(): @pytest.fixture(scope="function") -def model(config, dataset, generator, val_generator): +def gpu(): + return '0' + + +@pytest.fixture(scope="function") +def model(config, dataset, gpu, generator, val_generator): def create(): - return plugins.models.gan.ModelClass(config, dataset, generator, val_generator) + return plugins.models.gan.ModelClass(config, dataset, gpu, generator, val_generator) return create @@ -140,8 +145,8 @@ def test_gan(config, generator, val_generator): assert not gan.discriminator_fixed.trainable -def test_init(config, dataset, generator, val_generator): - dpmodel = plugins.models.gan.ModelClass(config, dataset, generator, val_generator) +def test_init(config, dataset, gpu, generator, val_generator): + dpmodel = plugins.models.gan.ModelClass(config, dataset, gpu, generator, val_generator) gan = plugins.models.gan.GAN(config, generator, val_generator) assert dpmodel.gan.__eq__(gan) assert isinstance(dpmodel.feature_model, keras.Model) diff --git a/tests/plugins/models/test_resnet.py b/tests/plugins/models/test_resnet.py index 8da2110..78bbaaf 100644 --- a/tests/plugins/models/test_resnet.py +++ b/tests/plugins/models/test_resnet.py @@ -82,6 +82,11 @@ def val_generator(): return deepprofiler.imaging.cropping.SingleImageCropGenerator +@pytest.fixture(scope="function") +def gpu(): + return '0' + + def test_define_model(config, dataset): config["train"]["model"]["name"] = "resnet" config["train"]["model"]["params"]["conv_blocks"] = 18 @@ -91,10 +96,10 @@ def test_define_model(config, dataset): assert isinstance(loss, str) or callable(loss) -def test_init(config, dataset, generator, val_generator): +def test_init(config, dataset, gpu, generator, val_generator): config["train"]["model"]["name"] = "resnet" config["train"]["model"]["params"]["conv_blocks"] = 18 - dpmodel = plugins.models.resnet.ModelClass(config, dataset, generator, val_generator) + dpmodel = plugins.models.resnet.ModelClass(config, dataset, gpu, generator, val_generator) model, optimizer, loss = plugins.models.resnet.define_model(config, dataset) assert dpmodel.feature_model.__eq__(model) assert dpmodel.optimizer.__eq__(optimizer) diff --git a/tests/plugins/models/test_vae.py b/tests/plugins/models/test_vae.py index 44d36b3..ece6c8c 100644 --- a/tests/plugins/models/test_vae.py +++ b/tests/plugins/models/test_vae.py @@ -82,6 +82,11 @@ def val_generator(): return deepprofiler.imaging.cropping.SingleImageCropGenerator +@pytest.fixture(scope="function") +def gpu(): + return '0' + + def test_define_model(config, dataset): vae, encoder, decoder, optimizer, loss = plugins.models.vae.define_model(config, dataset) assert isinstance(vae, keras.Model) @@ -91,8 +96,8 @@ def test_define_model(config, dataset): assert isinstance(loss, str) or callable(loss) -def test_init(config, dataset, generator, val_generator): - dpmodel = plugins.models.vae.ModelClass(config, dataset, generator, val_generator) +def test_init(config, dataset, gpu, generator, val_generator): + dpmodel = plugins.models.vae.ModelClass(config, dataset, gpu, generator, val_generator) vae, encoder, decoder, optimizer, loss = plugins.models.vae.define_model(config, dataset) assert dpmodel.feature_model.__eq__(vae) assert dpmodel.encoder.__eq__(encoder) From efb6949a64a7c36ab01d97acd0e944278bbae4c4 Mon Sep 17 00:00:00 2001 From: Nick Date: Tue, 24 Sep 2019 18:41:11 +0200 Subject: [PATCH 2/4] Config as mandatory parameter. --- deepprofiler/__main__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/deepprofiler/__main__.py b/deepprofiler/__main__.py index 62df75d..d0ccdc9 100644 --- a/deepprofiler/__main__.py +++ b/deepprofiler/__main__.py @@ -53,10 +53,7 @@ def cli(context, root, config, exp, cores, gpu): "features": root+"/outputs/" + exp + "/features/" } - context.obj["config"] = {} - context.obj["config"]["paths"] = {} - context.obj["config"]["paths"]["config"] = root+"/inputs/config/" + config - + config = dirs["config"] + "/" + config context.obj["cores"] = cores context.obj["gpu"] = gpu From bf69d94dc2d43d69f194350b4e6b1af0860743fd Mon Sep 17 00:00:00 2001 From: Nick Date: Tue, 24 Sep 2019 18:47:22 +0200 Subject: [PATCH 3/4] Gpu param for command line, tests. --- deepprofiler/learning/profiling.py | 2 +- .../repeat_channel_crop_generator.py | 49 +++++++++++++++++-- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/deepprofiler/learning/profiling.py b/deepprofiler/learning/profiling.py index bac2dd7..5fc89e1 100644 --- a/deepprofiler/learning/profiling.py +++ b/deepprofiler/learning/profiling.py @@ -11,7 +11,7 @@ class Profile(object): - def __init__(self, config, dset): + def __init__(self, config, dset, gpu): self.config = config self.dset = dset self.num_channels = len(self.config["dataset"]["images"]["channels"]) diff --git a/plugins/crop_generators/repeat_channel_crop_generator.py b/plugins/crop_generators/repeat_channel_crop_generator.py index 6258fcc..1685d9b 100644 --- a/plugins/crop_generators/repeat_channel_crop_generator.py +++ b/plugins/crop_generators/repeat_channel_crop_generator.py @@ -1,13 +1,36 @@ import numpy as np - +import tensorflow as tf import deepprofiler.imaging.cropping - +from keras.applications import inception_resnet_v2 def repeat_channels(crops, labels, repeats): + crops = crops[np.newaxis, :] + labels = labels[np.newaxis, :] + print('crops begin', crops.shape) + print('crops begin', labels.shape) + print(repeats) crops = np.reshape(crops, (crops.shape[0] * crops.shape[3], crops.shape[1], crops.shape[2], 1)) + print(crops.shape) crops = np.tile(crops, (1, 1, 1, repeats)) + print(crops.shape, crops.__class__) + print(labels.shape) labels = np.tile(labels, (repeats, 1)) + print(labels.shape) + #crops = tf.image.resize_images(crops, size=(299, 299)) + print(crops.shape, crops.__class__) return crops, labels + #return inception_resnet_v2.preprocess_input(crops), labels + #def crop_transform(crop_ph, image_size): + #crops_shape = crops.shape + #resized_crops = tf.image.resize_images(crops, size=(299, 299)) + #print(resized_crops.shape) + #reordered_channels = tf.transpose(resized_crops, [3, 0, 1, 2]) + #print(reordered_channels.shape) + #reshaped_data = tf.reshape(reordered_channels, shape=[-1, 299, 299, 1]) + #print(reshaped_data.shape) + #rgb_data = tf.image.grayscale_to_rgb(reordered_channels) + #print(rgb_data.shape) + #return inception_resnet_v2.preprocess_input(rgb_data), labels class GeneratorClass(deepprofiler.imaging.cropping.CropGenerator): @@ -21,6 +44,7 @@ def generate(self, sess, global_step=0): crops = data[0] labels = data[1] # TODO: enable multiple targets crops, labels = repeat_channels(crops, labels, self.config["dataset"]["images"]["channel_repeats"]) + crops = crops.eval(session=sess) global_step += 1 yield (crops, labels) @@ -30,5 +54,22 @@ class SingleImageGeneratorClass(deepprofiler.imaging.cropping.SingleImageCropGen def generate(self, session, global_step=0): crops = self.image_pool labels = self.label_pool - crops, labels = repeat_channels(crops, labels, self.config["dataset"]["images"]["channel_repeats"]) - yield [crops, labels] + #new_crops = np.array([[[[]]]]) + #new_labels = np.array([[[[]]]]) + for i in range(crops.shape[0]): + crop, label = repeat_channels(crops[i, :, :, :], labels[i, :], self.config["dataset"]["images"]["channel_repeats"]) + #crop = crop.eval(session=session) + crop = session.run(tf.image.resize_images(crop, size=(299, 299))) + if i == 0: + new_crops = crop + new_labels = label + else: + new_crops = np.concatenate((new_crops, crop), axis=0) + new_labels = np.concatenate((new_labels, label), axis=0) + + #print('last shape', crop.shape, label.shape) + #print('generator shapes', new_crops.shape, new_labels.shape) + #new_crops = new_crops.eval(session=session) + #crops, labels = repeat_channels(crops, labels, self.config["dataset"]["images"]["channel_repeats"]) + #crops = crops.eval(session=session) + yield [new_crops, new_labels] From 217e6729dff780ef6e4c87e158bc4494be61453f Mon Sep 17 00:00:00 2001 From: Nick Date: Tue, 24 Sep 2019 18:57:10 +0200 Subject: [PATCH 4/4] Gpu param tests --- deepprofiler/learning/profiling.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/deepprofiler/learning/profiling.py b/deepprofiler/learning/profiling.py index 5fc89e1..a3d8d56 100644 --- a/deepprofiler/learning/profiling.py +++ b/deepprofiler/learning/profiling.py @@ -24,7 +24,6 @@ def __init__(self, config, dset, gpu): .ModelClass(config, dset, gpu, self.crop_generator, self.profile_crop_generator) self.profile_crop_generator = self.profile_crop_generator(config, dset) - def configure(self): # Main session configuration configuration = tf.ConfigProto() @@ -45,7 +44,6 @@ def configure(self): self.config["profile"]["feature_layer"]).output) self.feat_extractor.summary() - def check(self, meta): output_folder = self.config["paths"]["features"] output_file = self.config["paths"]["features"] + "/{}_{}_{}.npz"