Merge pull request #742 from deeptools/develop

Develop

MANIFEST.in

@@ -0,0 +1,9 @@
+include hicexplorer/trained_models/10kb_model_cleanlab_proteins_obs_exp.BIN
+include hicexplorer/trained_models/25kb_model_cleanlab_proteins_obs_exp.BIN
+include hicexplorer/trained_models/50kb_model_cleanlab_proteins_obs_exp.BIN
+include hicexplorer/trained_models/100kb_model_cleanlab_proteins_obs_exp.BIN
+include hicexplorer/trained_models/10kb_model_cleanlab_proteins_range.BIN
+include hicexplorer/trained_models/25kb_model_cleanlab_proteins_range.BIN
+include hicexplorer/trained_models/50kb_model_cleanlab_proteins_range.BIN
+include hicexplorer/trained_models/100kb_model_cleanlab_proteins_range.BIN
+

azure-pipelines.yml

@@ -43,7 +43,9 @@ jobs:
     displayName: doc_test_and_number_of_cases
   - script: |
       source activate hicexplorer
-      py.test hicexplorer/test/general/ --doctest-modules --capture=sys -n 4
+      py.test hicexplorer/test/general/ --doctest-modules --capture=sys -n 4  --ignore=hicexplorer/test/general/test_hicTADClassifier.py --ignore=hicexplorer/test/general/test_hicTrainTADClassifier.py
+      py.test hicexplorer/test/general/test_hicTADClassifier.py
+      py.test hicexplorer/test/general/test_hicTrainTADClassifier.py
     displayName: pytest
 
 - job: 'OSX'
@@ -86,5 +88,7 @@ jobs:
     displayName: doc_test_and_number_of_cases
   - script: |
       source activate hicexplorer
-      py.test hicexplorer/test/general/ --doctest-modules --capture=sys -n 4 
+      py.test hicexplorer/test/general/ --doctest-modules --capture=sys -n 4  --ignore=hicexplorer/test/general/test_hicTADClassifier.py --ignore=hicexplorer/test/general/test_hicTrainTADClassifier.py
+      py.test hicexplorer/test/general/test_hicTADClassifier.py
+      py.test hicexplorer/test/general/test_hicTrainTADClassifier.py
     displayName: pytest

bin/chicExportData

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from hicexplorer.chicExportData import main
+
+if __name__ == "__main__":
+    main()
+

bin/hicInterIntraTAD

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from hicexplorer.hicInterIntraTAD import main
+
+if __name__ == "__main__":
+    main()

bin/hicTADClassifier

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from hicexplorer.hicTADClassifier import main
+
+if __name__ == "__main__":
+    main()

bin/hicTrainTADClassifier

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from hicexplorer.hicTrainTADClassifier import main
+
+if __name__ == "__main__":
+    main()

docs/conf.py

@@ -28,9 +28,12 @@
                 'matplotlib', 'matplotlib.pyplot', 'matplotlib.gridspec', 'matplotlib.ticker',
                 'matplotlib.textpath', 'matplotlib.patches', 'matplotlib.colors', 'matplotlib.cm',
                 'mpl_toolkits', 'mpl_toolkits.axisartist', 'mpl_toolkits.mplot3d', 'mpl_toolkits.axes_grid1',
-                'Bio', 'Bio.Seq', 'Bio.Alphabet', 'pyBigWig', 'tables', 'pytables', 'future', 'past', 'past.builtins',
+                'Bio.Seq', 'pyBigWig', 'tables', 'pytables', 'future', 'past', 'past.builtins',
                 'future.utils', 'cooler', 'logging', 'unidecode', 'hic2cool', 'hicmatrix', 'hicmatrix.HiCMatrix',
-                'hicmatrix.lib', 'krbalancing', 'fit_nbinom', 'pybedtools', 'graphviz', 'hyperopt', 'sklearn', 'sklearn.cluster']
+                'hicmatrix.lib', 'krbalancing', 'fit_nbinom', 'pybedtools', 'graphviz', 'hyperopt', 'sklearn', 'sklearn.cluster',
+                'h5py', 'numpy.random', 'sklearn', 'sklearn.model_selection', 'matplotlib.backends', 'sklearn.preprocessing',
+                'matplotlib.backends.backend_agg', 'cleanlab.classification', 'imblearn.under_sampling', 'sklearn.ensemble',
+                'sklearn.metrics', 'sklearn.impute', 'tarfile']
 
 for mod_name in MOCK_MODULES:
     sys.modules[mod_name] = Mock()

docs/content/News.rst

@@ -2,6 +2,41 @@ News and Developments
 =====================
 
 
+Release 3.7
+-----------
+**27 July 2021**
+
+- A new TAD prediction method: `hicTADClassifier` and a method to train own classifiers `hicTrainTADClassifier`. Thanks @AlbertLidel for the design and implementation.
+- New file formats for capture Hi-C modules. `chicViewpoint`, `chicSignificantInteractions`, `chicAggregateStatistic` and `chicDifferentialTest` use now HDF5 based files. A new export script `chicExportData` is provided to retrieve text files from these HDF5 files.
+- Implementing a few feature requests:
+    - `hicPlotMatrix`: TADs can be visualized with hicPlotMatrix
+    - `hicAdjustMatrix` is able to remove intra- or inter-chromosomal contacts (#664, #704)
+    - `hicValidateLocations`: An option to validate TADs and to use additional data stored in `cool` matrices
+    - `hicPCA`: Adding a function to select which eigenvector should be used for the output (#669)
+    - `hicConvertFormat`: Adding the function to export hicpro file format and to import `2D-text` files.
+    - `hicFindRestSites`: Support of multiple restriction cut sequences (#659)
+    - `hicPlotMatrix`: Option for loop locations spanning more than one bin to define if the start, center or end should be used for plotting (#640)
+    - `hicInterIntraTAD`: A new script to compute the ratio of inter and intra TAD. ($404)
+    - `hicAggregateContacts`: Option to consider the strand orientation (#633)
+    - `hicAverageRegions`: Option to consider the strand orientation (#633)
+    - `hicCompareMatrices`: An option to not normalize the matrices before the computation. (#677, #645) Thanks @lldelisle
+    - `hicDifferentialTAD`: Adding rank sum statistics to the output (#728, #727). Thanks @dawe
+    - `hicPlotDistVsCounts`: Adding a function to plot the counts vs distance in TAD regions. (#696) Thanks @lldelisle
+- Bug fixes:
+     - `hicCorrectMatrix`: A bug that lead to wrong correction factors for the KR correction for `cool` files (#724)
+     - `hicDifferentialTAD`: Solved multicore issue related to skipping data at the start and end of chromosomes (#725, #685)
+     - `hicHyperoptDetectLoops`: Added an option to set if the `chr` prefix should be added or removed (#723)
+     - `hicPCA`: Solving an issue if the region defined by the gene track is larger the region stored in the interaction matrix (#655, #710, #716, #719)
+     - `hicPCA`: Fixing a bug where the masking of bins was automatically applied which lead to differing matrix dimensions for the e.g. the Pearson correlation matrices (#618)
+     - `hicBuildMatrix`: Solving a bug if multiple restriction cut sites have the same dangling ends (#720)
+     - `hicBuildMatrix`: Solving a bug that the parameter `--removeSelfLigations` was always set to true. Changed parameter name to `--keepSelfLigations` to keep the functionality. If the parameter is not set, the self ligations are removed.
+     - `hicBuildMatrix`: If a region is specified, only the restrictionCutSite file information for that region is loaded to save memory (#646)
+     - `hicConvertFormat`: Fixing a bug to copy the genome annotation information in the case of a `cool` to `cool` file conversion (#657)
+     - `hicCorrelate`: Correcting the range of colors for the heatmap (#585)
+     - `hicCompartmentalization`: Fixed index bug (#635, #637) Thanks  @LeilyR
+- Updating `hicBuildMatrix` to be able to work with biopython versions > 1.77. Thanks @lldelisle (#691)
+
+
 Release 3.6
 -----------
 **10 November 2020**
@@ -113,7 +148,7 @@ Release 3.4.2
 
 Preprint
 --------
-**6 March 2020*
+**6 March 2020**
 
 The preprint of the loop detection algorithm is online via biorXiv: `<https://www.biorxiv.org/content/10.1101/2020.03.05.979096v1>`_