diff --git a/deepprofiler/__main__.py b/deepprofiler/__main__.py index 3c415ea..62df75d 100644 --- a/deepprofiler/__main__.py +++ b/deepprofiler/__main__.py @@ -22,7 +22,7 @@ type=click.Path("r")) @click.option("--config", default=None, help="Path to existing config file", - type=click.Path("r")) + type=click.STRING) @click.option("--cores", default=0, help="Number of CPU cores for parallel processing (all=0)", type=click.INT) @@ -52,13 +52,11 @@ def cli(context, root, config, exp, cores, gpu): "summaries": root+"/outputs/" + exp + "/summaries/", "features": root+"/outputs/" + exp + "/features/" } - if config is not None: - context.obj["config"] = {} - context.obj["config"]["paths"] = {} - context.obj["config"]["paths"]["config"] = config - dirs["config"] = os.path.dirname(os.path.abspath(config)) - else: - config = dirs["config"] + "/config.json" + + context.obj["config"] = {} + context.obj["config"]["paths"] = {} + context.obj["config"]["paths"]["config"] = root+"/inputs/config/" + config + context.obj["cores"] = cores context.obj["gpu"] = gpu diff --git a/plugins/models/autoencoder.py b/plugins/models/autoencoder.py index a7937f3..a1bad9d 100644 --- a/plugins/models/autoencoder.py +++ b/plugins/models/autoencoder.py @@ -66,6 +66,6 @@ def define_model(config, dset): class ModelClass(DeepProfilerModel): - def __init__(self, config, dset, generator, val_generator): - super(ModelClass, self).__init__(config, dset, generator, val_generator) + def __init__(self, config, dset, gpu, generator, val_generator): + super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator) self.feature_model, self.encoder, self.decoder, self.optimizer, self.loss = define_model(config, dset) diff --git a/plugins/models/cnn.py b/plugins/models/cnn.py index 3b3245f..813f27c 100644 --- a/plugins/models/cnn.py +++ b/plugins/models/cnn.py @@ -52,6 +52,6 @@ def define_model(config, dset): class ModelClass(DeepProfilerModel): - def __init__(self, config, dset, generator, val_generator): - super(ModelClass, self).__init__(config, dset, generator, val_generator) + def __init__(self, config, dset, gpu, generator, val_generator): + super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator) self.feature_model, self.optimizer, self.loss = define_model(config, dset) diff --git a/plugins/models/densenet.py b/plugins/models/densenet.py index 188f12b..fdf5d1f 100644 --- a/plugins/models/densenet.py +++ b/plugins/models/densenet.py @@ -47,7 +47,7 @@ def define_model(config, dset): return model, optimizer, loss_func class ModelClass(DeepProfilerModel): - def __init__(self, config, dset, generator, val_generator): - super(ModelClass, self).__init__(config, dset, generator, val_generator) + def __init__(self, config, dset, gpu, generator, val_generator): + super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator) self.feature_model, self.optimizer, self.loss = define_model(config, dset) diff --git a/plugins/models/gan.py b/plugins/models/gan.py index 1ef9b8c..c8f6280 100644 --- a/plugins/models/gan.py +++ b/plugins/models/gan.py @@ -140,8 +140,8 @@ def train(self, epochs, steps_per_epoch, init_epoch): class ModelClass(DeepProfilerModel): - def __init__(self, config, dset, generator, val_generator): - super(ModelClass, self).__init__(config, dset, generator, val_generator) + def __init__(self, config, dset, gpu, generator, val_generator): + super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator) self.gan = GAN(config, self.train_crop_generator, self.val_crop_generator) self.feature_model = self.gan.discriminator diff --git a/plugins/models/vae.py b/plugins/models/vae.py index 5d4f939..f2df4aa 100644 --- a/plugins/models/vae.py +++ b/plugins/models/vae.py @@ -88,6 +88,6 @@ def vae_loss(x, x_decoded_mean): class ModelClass(DeepProfilerModel): - def __init__(self, config, dset, generator, val_generator): - super(ModelClass, self).__init__(config, dset, generator, val_generator) + def __init__(self, config, dset, gpu, generator, val_generator): + super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator) self.feature_model, self.encoder, self.generator, self.optimizer, self.loss = define_model(config, dset) diff --git a/tests/deepprofiler/learning/test_model.py b/tests/deepprofiler/learning/test_model.py index 7ce836f..b202480 100644 --- a/tests/deepprofiler/learning/test_model.py +++ b/tests/deepprofiler/learning/test_model.py @@ -111,19 +111,22 @@ def val_crop_generator(config): generator = module.SingleImageGeneratorClass return generator +@pytest.fixture(scope="function") +def gpu(): + return '0' @pytest.fixture(scope="function") -def model(config, dataset, crop_generator, val_crop_generator): +def model(config, dataset, gpu, crop_generator, val_crop_generator): def create(): module = importlib.import_module("plugins.models.{}".format(config["train"]["model"]["name"])) importlib.invalidate_caches() - dpmodel = module.ModelClass(config, dataset, crop_generator, val_crop_generator) + dpmodel = module.ModelClass(config, dataset, gpu, crop_generator, val_crop_generator) return dpmodel return create -def test_init(config, dataset, crop_generator, val_crop_generator): - dpmodel = DeepProfilerModel(config, dataset, crop_generator, val_crop_generator) +def test_init(config, dataset, gpu, crop_generator, val_crop_generator): + dpmodel = DeepProfilerModel(config, dataset, gpu, crop_generator, val_crop_generator) assert dpmodel.feature_model is None assert dpmodel.config == config assert dpmodel.dset == dataset diff --git a/tests/deepprofiler/learning/test_profiling.py b/tests/deepprofiler/learning/test_profiling.py index b9a3e9b..b0ce1c7 100644 --- a/tests/deepprofiler/learning/test_profiling.py +++ b/tests/deepprofiler/learning/test_profiling.py @@ -98,6 +98,12 @@ def locations(out_dir, metadata, config, make_struct): }) locs.to_csv(path, index=False) + +@pytest.fixture(scope="function") +def gpu(): + return '0' + + @pytest.fixture(scope="function") def checkpoint(config, dataset): crop_generator = importlib.import_module( @@ -107,7 +113,7 @@ def checkpoint(config, dataset): "plugins.crop_generators.{}".format(config["train"]["model"]["crop_generator"])) \ .SingleImageGeneratorClass dpmodel = importlib.import_module("plugins.models.{}".format(config["train"]["model"]["name"])) \ - .ModelClass(config, dataset, crop_generator, profile_crop_generator) + .ModelClass(config, dataset, gpu, crop_generator, profile_crop_generator) dpmodel.feature_model.compile(dpmodel.optimizer, dpmodel.loss) filename = os.path.join(config["paths"]["checkpoints"], config["profile"]["checkpoint"]) dpmodel.feature_model.save_weights(filename) @@ -115,12 +121,12 @@ def checkpoint(config, dataset): @pytest.fixture(scope="function") -def profile(config, dataset): - return deepprofiler.learning.profiling.Profile(config, dataset) +def profile(config, dataset, gpu): + return deepprofiler.learning.profiling.Profile(config, dataset, gpu) -def test_init(config, dataset): - prof = deepprofiler.learning.profiling.Profile(config, dataset) +def test_init(config, dataset, gpu): + prof = deepprofiler.learning.profiling.Profile(config, dataset, gpu) test_num_channels = len(config["dataset"]["images"]["channels"]) assert prof.config == config assert prof.dset == dataset @@ -152,8 +158,8 @@ def test_extract_features(profile, metadata, locations, checkpoint): assert os.path.isfile(output_file) -def test_profile(config, dataset, data, locations, checkpoint): - deepprofiler.learning.profiling.profile(config, dataset) +def test_profile(config, dataset, data, locations, checkpoint, gpu): + deepprofiler.learning.profiling.profile(config, dataset, gpu) for index, row in dataset.meta.data.iterrows(): output_file = config["paths"]["features"] + "/{}_{}_{}.npz" \ .format(row["Metadata_Plate"], row["Metadata_Well"], row["Metadata_Site"]) diff --git a/tests/deepprofiler/learning/test_training.py b/tests/deepprofiler/learning/test_training.py index e44f7e5..06165e2 100644 --- a/tests/deepprofiler/learning/test_training.py +++ b/tests/deepprofiler/learning/test_training.py @@ -98,13 +98,14 @@ def locations(out_dir, metadata, config, make_struct): def test_learn_model(config, dataset, data, locations, out_dir, make_struct): epoch = 1 verbose = 1 - deepprofiler.learning.training.learn_model(config, dataset, epoch, verbose=verbose) + gpu = '0' + deepprofiler.learning.training.learn_model(config, dataset, gpu, epoch, verbose=verbose) assert os.path.exists(os.path.join(config["paths"]["checkpoints"], "checkpoint_0001.hdf5")) assert os.path.exists(os.path.join(config["paths"]["checkpoints"], "checkpoint_0002.hdf5")) assert os.path.exists(os.path.join(config["paths"]["logs"], "log.csv")) epoch = 3 config["train"]["model"]["epochs"] = 4 - deepprofiler.learning.training.learn_model(config, dataset, epoch, verbose=verbose) + deepprofiler.learning.training.learn_model(config, dataset, gpu, epoch, verbose=verbose) assert os.path.exists(os.path.join(config["paths"]["checkpoints"], "checkpoint_0003.hdf5")) assert os.path.exists(os.path.join(config["paths"]["checkpoints"], "checkpoint_0004.hdf5")) assert os.path.exists(os.path.join(config["paths"]["logs"], "log.csv")) diff --git a/tests/plugins/models/test_autoencoder.py b/tests/plugins/models/test_autoencoder.py index 82f7273..fa0a5f0 100644 --- a/tests/plugins/models/test_autoencoder.py +++ b/tests/plugins/models/test_autoencoder.py @@ -82,6 +82,10 @@ def val_generator(): return deepprofiler.imaging.cropping.SingleImageCropGenerator +@pytest.fixture(scope="function") +def gpu(): + return '0' + def test_define_model(config, dataset): autoencoder, encoder, decoder, optimizer, loss = plugins.models.autoencoder.define_model(config, dataset) assert isinstance(autoencoder, keras.Model) @@ -91,8 +95,8 @@ def test_define_model(config, dataset): assert isinstance(loss, str) or callable(loss) -def test_init(config, dataset, generator, val_generator): - dpmodel = plugins.models.autoencoder.ModelClass(config, dataset, generator, val_generator) +def test_init(config, dataset, gpu, generator, val_generator): + dpmodel = plugins.models.autoencoder.ModelClass(config, dataset, gpu, generator, val_generator) autoencoder, encoder, decoder, optimizer, loss = plugins.models.autoencoder.define_model(config, dataset) assert dpmodel.feature_model.__eq__(autoencoder) assert dpmodel.encoder.__eq__(encoder) diff --git a/tests/plugins/models/test_cnn.py b/tests/plugins/models/test_cnn.py index 961db9b..e79e094 100644 --- a/tests/plugins/models/test_cnn.py +++ b/tests/plugins/models/test_cnn.py @@ -82,6 +82,11 @@ def val_generator(): return deepprofiler.imaging.cropping.SingleImageCropGenerator +@pytest.fixture(scope="function") +def gpu(): + return '0' + + def test_define_model(config, dataset): model, optimizer, loss = plugins.models.cnn.define_model(config, dataset) assert isinstance(model, keras.Model) @@ -89,8 +94,8 @@ def test_define_model(config, dataset): assert isinstance(loss, str) or callable(loss) -def test_init(config, dataset, generator, val_generator): - dpmodel = plugins.models.cnn.ModelClass(config, dataset, generator, val_generator) +def test_init(config, dataset, gpu, generator, val_generator): + dpmodel = plugins.models.cnn.ModelClass(config, dataset, gpu, generator, val_generator) model, optimizer, loss = plugins.models.cnn.define_model(config, dataset) assert dpmodel.feature_model.__eq__(model) assert dpmodel.optimizer.__eq__(optimizer) diff --git a/tests/plugins/models/test_gan.py b/tests/plugins/models/test_gan.py index 0a8ccd9..938c4ff 100644 --- a/tests/plugins/models/test_gan.py +++ b/tests/plugins/models/test_gan.py @@ -108,9 +108,14 @@ def val_generator(): @pytest.fixture(scope="function") -def model(config, dataset, generator, val_generator): +def gpu(): + return '0' + + +@pytest.fixture(scope="function") +def model(config, dataset, gpu, generator, val_generator): def create(): - return plugins.models.gan.ModelClass(config, dataset, generator, val_generator) + return plugins.models.gan.ModelClass(config, dataset, gpu, generator, val_generator) return create @@ -140,8 +145,8 @@ def test_gan(config, generator, val_generator): assert not gan.discriminator_fixed.trainable -def test_init(config, dataset, generator, val_generator): - dpmodel = plugins.models.gan.ModelClass(config, dataset, generator, val_generator) +def test_init(config, dataset, gpu, generator, val_generator): + dpmodel = plugins.models.gan.ModelClass(config, dataset, gpu, generator, val_generator) gan = plugins.models.gan.GAN(config, generator, val_generator) assert dpmodel.gan.__eq__(gan) assert isinstance(dpmodel.feature_model, keras.Model) diff --git a/tests/plugins/models/test_resnet.py b/tests/plugins/models/test_resnet.py index 8da2110..78bbaaf 100644 --- a/tests/plugins/models/test_resnet.py +++ b/tests/plugins/models/test_resnet.py @@ -82,6 +82,11 @@ def val_generator(): return deepprofiler.imaging.cropping.SingleImageCropGenerator +@pytest.fixture(scope="function") +def gpu(): + return '0' + + def test_define_model(config, dataset): config["train"]["model"]["name"] = "resnet" config["train"]["model"]["params"]["conv_blocks"] = 18 @@ -91,10 +96,10 @@ def test_define_model(config, dataset): assert isinstance(loss, str) or callable(loss) -def test_init(config, dataset, generator, val_generator): +def test_init(config, dataset, gpu, generator, val_generator): config["train"]["model"]["name"] = "resnet" config["train"]["model"]["params"]["conv_blocks"] = 18 - dpmodel = plugins.models.resnet.ModelClass(config, dataset, generator, val_generator) + dpmodel = plugins.models.resnet.ModelClass(config, dataset, gpu, generator, val_generator) model, optimizer, loss = plugins.models.resnet.define_model(config, dataset) assert dpmodel.feature_model.__eq__(model) assert dpmodel.optimizer.__eq__(optimizer) diff --git a/tests/plugins/models/test_vae.py b/tests/plugins/models/test_vae.py index 44d36b3..ece6c8c 100644 --- a/tests/plugins/models/test_vae.py +++ b/tests/plugins/models/test_vae.py @@ -82,6 +82,11 @@ def val_generator(): return deepprofiler.imaging.cropping.SingleImageCropGenerator +@pytest.fixture(scope="function") +def gpu(): + return '0' + + def test_define_model(config, dataset): vae, encoder, decoder, optimizer, loss = plugins.models.vae.define_model(config, dataset) assert isinstance(vae, keras.Model) @@ -91,8 +96,8 @@ def test_define_model(config, dataset): assert isinstance(loss, str) or callable(loss) -def test_init(config, dataset, generator, val_generator): - dpmodel = plugins.models.vae.ModelClass(config, dataset, generator, val_generator) +def test_init(config, dataset, gpu, generator, val_generator): + dpmodel = plugins.models.vae.ModelClass(config, dataset, gpu, generator, val_generator) vae, encoder, decoder, optimizer, loss = plugins.models.vae.define_model(config, dataset) assert dpmodel.feature_model.__eq__(vae) assert dpmodel.encoder.__eq__(encoder)