From 231f02471eafe5e0750d6954cc6cfd66c211ae1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Pavlin=20Poli=C4=8Dar?= <pavlin.g.p@gmail.com>
Date: Mon, 2 Sep 2019 17:18:52 +0200
Subject: [PATCH] Load data: Add data loader for H5AD format

---
 .../single_cell/tests/data/data_dense.h5ad    | Bin 0 -> 11060 bytes
 .../single_cell/tests/data/data_sparse.h5ad   | Bin 0 -> 19560 bytes
 .../single_cell/tests/test_load_data.py       |  20 +++++-
 .../single_cell/tests/test_owloaddata.py      |  41 ++++++++++++-
 .../single_cell/widgets/load_data.py          |  58 +++++++++++++++++-
 setup.py                                      |   1 +
 6 files changed, 116 insertions(+), 4 deletions(-)
 create mode 100644 orangecontrib/single_cell/tests/data/data_dense.h5ad
 create mode 100644 orangecontrib/single_cell/tests/data/data_sparse.h5ad

diff --git a/orangecontrib/single_cell/tests/data/data_dense.h5ad b/orangecontrib/single_cell/tests/data/data_dense.h5ad
new file mode 100644
index 0000000000000000000000000000000000000000..96db6ba6fab32067826806adce1e8305c601e242
GIT binary patch
literal 11060
zcmeHNOLJ6J5bj9=Mid1^5PUHBKzt`5JY0EGJQtQw%7P6AL8mlTF{F}cfg4%kj-TMd
zg$oz{1ePxON7~=%{?7EdIWt3*T9(B<)Sd4B`a6B^>D!mwJ~KaEI(K3BuEV<oV|G?%
zWKUB0@u!c)_Y9#;|4mnVF7Lbib;^{U;Gf7#j_>vOZQj+-&z~1%GCrkGlernw7>9T#
zpNtG#I=^%UE4}8+J?`=kuQNBZ|MsoT1bx4}k>q!4X0uM`IAMb9$<9>2_JInuGxw$}
z(te+R$YtA}*2)KSZ~ursTNr&gI4cK#OuEPpnN9wwDGNC3Ps(&p9cesz%Be~0a~LtN
zuYY@8<eTXJ*Yz(xyL_&vEAfky{`B}$@_&;%JI3?+?tbX2wd{)Qf72X`$4v71|9z)V
z_@9cN-g&I?xG%Kl_r2FMfOSMUEe{j_fad!}owu^~)!@!!to@M2!#%)Zk!NLf`PN`n
z9|zu>QDOGzP0n*xZfz`Y+`X~6^1}f9y`v^MKDpj4^swF^J#SNg@r;<&uJ`TFqg4d+
z#^;c7S}s}MG%D8nck+fS8a}V>>)$$RSNve?^^0E5KRHJIeQiHC6VBO)-1}rZ2N;j!
z9l?t;N5zR8=moD)K#zz;UA<`#qa>bW<MSy2sy=GeVq{F`WnBY6k4KD2JYvv^Lvj#<
z9dY_8qi>W>F7w9N<VQ(8&ryZ^h!uJ>wDNpl0Ut89uP4tT$>+uMk`pn|iF0ZVBZqR^
zbv45{?f8sKJjYN+#HhFV(WVX=`Qu{7IazN^V~jYToQP!|^LUU58>6u!bJdKxnu!xJ
z=uMM6=>v8==!qC~jy4{B<cyPTK5fc47g)lZAzI@h51BipX07l!7tVI9woOhAhmWI(
zuORuO@HDUC>><Cv)z3IXrfCtUP6-C|_Q4p$+cl8<&G4M$0f%0b5le(dKXK^b9J5AW
z5B=ljki(eq+LEs!!_Ny_5Axw1rNh@AMf_&y%`u3%Bp!1b^FV)FrpOwjiYI6IP@Xh+
z`PlN8Gk<sNFGcVLn^A)M3|udm9;IaZJ7ZHY@#d!ID4EY0I9D)t8(5y`*&6d11Aktc
zQiUtRHig>v3h-$Ir2x^NDDz4IqPIY}0z@OBU{iqTRup_tfau@0#YF$3;LU=m0KhUP
z?t+5%moA>Y{3#5+fG6FYKxO{pBtTAsa{i3jY`HRi+Hl_@$@At<8P3)#^G_Pi7AoV5
zhO^<!_zA<=^JRR&aNGcljL(~Gn@RznGaR=TgX{L?3hDrZxGz^wG#JEvxq=GAAnwZ*
zZzMAA%VoMV?#s=+4CDB6#h;4I_vMN|LK*kv3VSui@#Us|4cE!MUL33q`U{J40HBT%
zKOqMVZs7}g53S_S3wxaspOeG2I9gmUSMf9QN)<mXuU7F>@>&%?DMzaKqV%iy2{~HD
z7v%LSJ}+-n@j3CW0^PB(k@Bp?^_JTOuElk%hHG)XUBk7wj@NK4u6Js<7T3EqT#L)M
zQp@Yq;(EWv*W&u1hHG(s=s2IJvxC*u{?)tdgT7VpM<e}K$*j-q^I6dr;2H@YuM=Gc
zx(sv~=rYh{pvyp)fi44G2D%J9eg@Eoa7P30_R&TU5N-4d(MC@ZZS)?|MvoG0^fJ*t
z*0RwXML&8d^pU(i^jgtIPZn+TZqe?yZ1jTBkDf8w=q;m-9yHqMRillbHrnWYqm6F^
zqm5oV+UU8Xjqg$?eW%F%mZ7xo^lRU(YPd1q5zgRt>R{((R{J|nkvr!2k3J{;7Uwq<
zkq+($nbmzhDH&-07N_qEzv^|tJlO9HZhX1CHTe4W#>!yR9o!$|{vP?0r2Y0J|1d^o
jxDhmIV(0HN&}E>@z=#ZBdz0@w_{$t^?4d>*d#-;0@Q`G2

literal 0
HcmV?d00001

diff --git a/orangecontrib/single_cell/tests/data/data_sparse.h5ad b/orangecontrib/single_cell/tests/data/data_sparse.h5ad
new file mode 100644
index 0000000000000000000000000000000000000000..db3ea00bd0bb17718a2981cb601cb6c034bfc9dc
GIT binary patch
literal 19560
zcmeHPOK%)S5U%y&u!Mxz5c1-Y&D-Q@JI(`%$69$=AxbRp0S-~JNjBJ$lUQDZ5;%|x
zk~tvdgg9~lDM#*HIOLEU*Bm+UKcK$p`eu82dv{rg6xr#vdV8w6y1MF{@@!A_%zig@
z`t16(+tvz7r6MoLP<F_Vzpcz(a2(R?ziJ01YnQEkIN%OSf<KWh#veBQq1?>JXU>R9
zQa@W8CVvjPBkCbPwa*6vQ)ecp;pwO5?ufOIEzXS0`|n(9W~?vki&^`{ZAUXsM>(KJ
zTav*-{0>?Cj=bmwKC$t=;<vWL38wh%0r)EI4&%3KelFW}jc(tlFV%58cl*ZN^+prz
z!jgtC<Mt#4|7H8h#Y?A4dR4EOe6{QjsBg;XS+j#z|H}K_C#YKY)v2PsWuqUScYMZs
z?W_%0p$)&cLcG?=l%k9CGb9@o<=JD+h5BN%apmTn#as0y*;+!6WYuBr&IIw#;J1?l
z-a45!eH-+6OG{sKw!Y9%Ju2(OYbvets(ut+2H9@RxFTDvbr$4AesJ#8r3;w`JMjg(
zRO))uT|cRp?by3cg@sQ(|Cj|RFhR-XPdJ+4p7W18{u<Y`<c~RqU1nSS3CB0cTKuDq
z|5B)D+%en&$Fpb5@m~(*?TcZZ@rj4IE9(P3{~uoo_~x!`2>9l%eT%gIugzT<4&=>U
zc{SjhyYgDVH+MA_U7`KvuDl-b&0X1KeBynsal26+pODQ_{?U9rSCk*m<E_XalWkEw
zqw+?Kufg?Zlz&p*it<m$+fn{;*&gK|lWLSdAv>b{qp~x~AD3NG{+R5J@<(Nl@fq(K
z&Kg{MV|)#+eKEcU*GP=7!SzmzuferH#@FCF5aVlb9gOicxDLhm8eE5Cd=0K6#^?Xp
zsmA<#b>`keqv|4<@z>t(BAId9-p?L4L5%`7UuqHp34w$_LLecK5J(6l1QG%Xfp!E+
zvc~0>>_)L<cXrGx{9!|wk1uOKM2D-}vpj|9oz&)vZ2rdN1~of@{(-VnTQPm*fq@~*
zML;Ff1A1TU?&LA%^1`jlB6m&GvuTjW#f}C2W&0iz9f4WPyfHLB@yZ<x<P(_?NC+eZ
z5&{W<guu#2!0oS<Z55;bZ0YXPkaaIJA=^QmY92Gu-$8B_v1=mtKP*=1!0W`vc=oaN
zd7=J(XkTBqag;;L<9=)i$90>l-#*_Se=O$=Tgv$3YLRn7xqe&fW$TZR@%;YqnDS_O
z9LjW}Yvxn0asNd(b#@yYlc{y^gR=0&>CEo8l)#=F?xwH@itP~WVR9c3`>yQ6UMTi4
zv8Tv>>cDLdWHEMFEA|&5gMC0=lOPKj>|t>p=)<**V<3ZjAlL+5=*J%E2Cb0e+}Mxg
z+^`$6l;d6{^k6RN@_mCY*ao|x1GW<b_z2r*6Xt_H>~s3QU<~b~FPI;GU=01Ie#pXi
zVn}}=4_kcw14`3w+C$%H7smK@5ewRdxu6>|u-oV3eEPUj56=_lr(c|lc%dKqp$9Us
zjdo!!`b;0f_@OWUxg@q6>+9#a^>KyGv=iqJI5A#e3-PCK%E~aHPdrzUBSw_L+^`*X
z^L)`q+6h_oV=nAP@6kG{^@P?*trxX^ruC-Qrq-{ueyjC+tv_h}MeDCxf7AL%EAIbc
zTc%W-v4DndzyFBRsUX(+%|i`k@rmbQ@INV1%|5&2pME@$No1$U4N<y1|8(C;J>$=Q
z^H7fvxL!dVqU4S`GJPfl5&{W<gg`<dA+U-N=(T>r+8BlVS!G@EVvXL%`sv9;rXgEq
z=gW$fwtkv%KD2-G!Aa{OUb&;~BN8VekPt`+Bm@!y34s-k0N?W*!J<e@_#6~+rzmX8
z@_V*^+N23ttQGV<y~f>4qWeeeeX#lZwZ^<A7m<VPR<Krc=2c{V?%HB~@!pl@+*b{Z
zL5{dv>G{0YfK#16uT`pj;Pm}$h&U=-?`54xplY!-Q+3j38el~xvb9<k3X(_*Ybi>a
zU$ufHfYLdNlE}m>bF?6dbgwdt6Cr2bCW)l8GJl#xHk#R6WNMYZUtl7Y450!O`x45W
zE--;hn<TQ_%B&S6k)~GWg939frV|TNmAqPz#I8V_PGx|(JPGCW=&&C8^r3)07|_U>
z`tth&8p~$Ce<Yyy1@zv4M%ogH(7xRPy(^%12K0`At_Jk>fW~`b{(N|!%%}0*nNMR+
zz^CyZn@{7JMxVysice#o!Kbl5luAM%A&?MQ4G8qgJJ~be`Fp`-eP3R`W@VFp;A{N<
zteb#ezTelje)$W}M`(F~>GBe;UqtPBnxz*W_@+i_weETP*t!sMo&B4!j1_j2TczE<
zf3qrWjepul9e@AFVCU~d^!ojj>POBpEU8j?{t)Q&`zepl7qM{nQdh3mml~hnS)9{f
dP;rG9fs;k?o|JpOv&7}8OrPsHBnd<i_y@P2?9~7O

literal 0
HcmV?d00001

diff --git a/orangecontrib/single_cell/tests/test_load_data.py b/orangecontrib/single_cell/tests/test_load_data.py
index 64ca96b5..8897c7a6 100644
--- a/orangecontrib/single_cell/tests/test_load_data.py
+++ b/orangecontrib/single_cell/tests/test_load_data.py
@@ -9,7 +9,7 @@
 
 from orangecontrib.single_cell.widgets.load_data import (
     LoomLoader, ExcelLoader, MtxLoader, CountLoader, Loader, PickleLoader,
-    CsvLoader, get_data_loader, Concatenate
+    CsvLoader, H5ADLoader, get_data_loader, Concatenate,
 )
 
 
@@ -21,6 +21,8 @@ def test_get_data_loader(self):
         self.assertIsInstance(loader, Loader)
         self.assertIsInstance(get_data_loader("data.xls"), ExcelLoader)
         self.assertIsInstance(get_data_loader("data.loom"), LoomLoader)
+        self.assertIsInstance(get_data_loader("data_sparse.h5ad"), H5ADLoader)
+        self.assertIsInstance(get_data_loader("data_dense.h5ad"), H5ADLoader)
 
     def test_get_data_loader_pickle(self):
         self.assertIsInstance(get_data_loader("data.pkl"), PickleLoader)
@@ -89,6 +91,22 @@ def test_file_summary_loom(self):
         self.assertEqual(loader.n_cols, 20)
         self.assertEqual(round(loader.sparsity, 2), 0.93)
 
+    def test_file_summary_h5ad_sparse(self):
+        file_name = os.path.join(os.path.dirname(__file__), "data/data_sparse.h5ad")
+        loader = H5ADLoader(file_name)
+        self.assertEqual(19560, loader.file_size)
+        self.assertEqual(20, loader.n_rows)
+        self.assertEqual(25, loader.n_cols)
+        self.assertEqual(0.59, round(loader.sparsity, 2))
+
+    def test_file_summary_h5ad_dense(self):
+        file_name = os.path.join(os.path.dirname(__file__), "data/data_dense.h5ad")
+        loader = H5ADLoader(file_name)
+        self.assertEqual(11060, loader.file_size)
+        self.assertEqual(20, loader.n_rows)
+        self.assertEqual(25, loader.n_cols)
+        self.assertEqual(0.59, round(loader.sparsity, 2))
+
     def test_load_data_mtx(self):
         file_name = os.path.join(os.path.dirname(__file__),
                                  "data/10x/mm10/matrix.mtx")
diff --git a/orangecontrib/single_cell/tests/test_owloaddata.py b/orangecontrib/single_cell/tests/test_owloaddata.py
index 169cd8ef..0eeec42a 100644
--- a/orangecontrib/single_cell/tests/test_owloaddata.py
+++ b/orangecontrib/single_cell/tests/test_owloaddata.py
@@ -5,7 +5,6 @@
 import pandas as pd
 
 from Orange.data import ContinuousVariable, Variable
-from Orange.widgets.data.owtable import OWDataTable
 from Orange.widgets.tests.base import WidgetTest
 
 from orangecontrib.single_cell.widgets.owloaddata import OWLoadData
@@ -241,6 +240,46 @@ def test_load_data_loom_sample(self):
         self.assertListEqual([attr.name for attr in data.domain.attributes],
                              ["0", "1", "2", "3", "6", "7"])
 
+    def test_load_data_h5ad_sparse_sample(self):
+        file_name = os.path.join(self._path, "data_sparse.h5ad")
+        self.widget.set_current_path(file_name)
+        self.widget.sample_rows_cb.setChecked(True)
+        self.widget.sample_cols_cb.setChecked(True)
+        self.widget.set_sample_rows_p(30)
+        self.widget.set_sample_cols_p(10)
+        self.widget.commit()
+        data = self.get_output("Data")
+        X = np.array([[0, 2, 1, 0], [1, 0, 0, 0], [2, 1, 2, 1], [1, 1, 0, 0],
+                      [1, 0, 0, 0], [0, 1, 0, 2], [1, 0, 0, 1], [0, 2, 0, 1],
+                      [0, 1, 1, 0]])
+        npt.assert_array_equal(data.X, X)
+        metas = np.array([[0, 0, 10], [1, 0, 10], [2, 0, 15], [3, 0, 10],
+                          [6, 1, 11], [7, 1, 15], [8, 1, 11], [9, 1, 20],
+                          [14, 2, 12]])
+        npt.assert_array_equal(data.metas, metas)
+        self.assertListEqual([attr.name for attr in data.domain.attributes],
+                             ["Gene 0", "Gene 1", "Gene 2", "Gene 3"])
+
+    def test_load_data_h5ad_dense_sample(self):
+        file_name = os.path.join(self._path, "data_dense.h5ad")
+        self.widget.set_current_path(file_name)
+        self.widget.sample_rows_cb.setChecked(True)
+        self.widget.sample_cols_cb.setChecked(True)
+        self.widget.set_sample_rows_p(30)
+        self.widget.set_sample_cols_p(10)
+        self.widget.commit()
+        data = self.get_output("Data")
+        X = np.array([[0, 2, 1, 0], [1, 0, 0, 0], [2, 1, 2, 1], [1, 1, 0, 0],
+                      [1, 0, 0, 0], [0, 1, 0, 2], [1, 0, 0, 1], [0, 2, 0, 1],
+                      [0, 1, 1, 0]])
+        npt.assert_array_equal(data.X, X)
+        metas = np.array([[0, 0, 10], [1, 0, 10], [2, 0, 15], [3, 0, 10],
+                          [6, 1, 11], [7, 1, 15], [8, 1, 11], [9, 1, 20],
+                          [14, 2, 12]])
+        npt.assert_array_equal(data.metas, metas)
+        self.assertListEqual([attr.name for attr in data.domain.attributes],
+                             ["Gene 0", "Gene 1", "Gene 2", "Gene 3"])
+
     def test_not_enough_headers(self):
         file_name = os.path.join(self._path, "DATA_MATRIX_LOG_TPM.txt")
         self.widget.set_current_path(file_name)
diff --git a/orangecontrib/single_cell/widgets/load_data.py b/orangecontrib/single_cell/widgets/load_data.py
index 5654352f..291d3670 100644
--- a/orangecontrib/single_cell/widgets/load_data.py
+++ b/orangecontrib/single_cell/widgets/load_data.py
@@ -6,11 +6,12 @@
 import numpy as np
 import pandas as pd
 import scipy.io
+import scipy.sparse as sp
 import loompy as lp
 import xlrd
 
 from Orange.data import (
-    ContinuousVariable, DiscreteVariable, Domain, Table, StringVariable
+    ContinuousVariable, DiscreteVariable, Domain, Table
 )
 from Orange.data.io import (
     Compression, open_compressed, PickleReader,
@@ -49,6 +50,8 @@ def get_data_loader(file_name):
         return ExcelLoader(file_name)
     elif ext == ".loom":
         return LoomLoader(file_name)
+    elif ext == ".h5ad":
+        return H5ADLoader(file_name)
     else:
         return Loader(file_name)
 
@@ -425,7 +428,7 @@ def __into_orange_table(self, attrs, X, meta_parts):
     @staticmethod
     def __guess_metas(meta_parts):
         def guessed_var(i, var_name, dtype):
-            if np.issubdtype(dtype, np.number):
+            if pd.core.dtypes.common.is_numeric_dtype(dtype):
                 return ContinuousVariable.make(var_name)
             orig_values = M[:, i]
             val_map, values, var_type = guess_data_type(orig_values)
@@ -666,6 +669,57 @@ def _load_data(self, skip_row=None, skip_col=None, **kwargs):
             attrs = [ContinuousVariable.make(str(g)) for g in gene_names]
             meta_df = pd.DataFrame({key: ds.ca[key][self._use_cols_mask]
                                     for key in ds.ca.keys()})
+
+        return attrs, X, meta_df, meta_df.index
+
+
+class H5ADLoader(Loader):
+    def __init__(self, file_name):
+        super().__init__(file_name)
+        self.header_rows_count = 0
+        self.header_cols_count = 0
+        self.FIXED_FORMAT = False
+        self.ENABLE_ANNOTATIONS = False
+        self.transposed = False
+        self.row_annotations_enabled = False
+        self.col_annotations_enabled = False
+
+    def _set_file_parameters(self):
+        import anndata
+        try:
+            adata = anndata.read_h5ad(self._file_name)
+            self.n_rows, self.n_cols = adata.shape
+            all_el = self.n_rows * self.n_cols
+            if sp.issparse(adata.X):
+                self.sparsity = (all_el - adata.X.tocsr().count_nonzero()) / all_el
+            else:
+                self.sparsity = (all_el - np.count_nonzero(adata.X)) / all_el
+        except OSError:
+            pass
+
+    def _load_data(self, skip_row=None, skip_col=None, **kwargs):
+        import anndata
+        adata = anndata.read_h5ad(self._file_name)
+
+        if skip_row is not None:
+            mask = np.array([not skip_row(i) for i in range(adata.shape[0])])
+            self._use_rows_mask = mask
+        else:
+            self._use_rows_mask = np.ones(adata.shape[0], dtype=bool)
+        if skip_col is not None:
+            mask = np.array([not skip_col(i) for i in range(adata.shape[1])])
+            self._use_cols_mask = mask
+        else:
+            self._use_cols_mask = np.ones(adata.shape[1], dtype=bool)
+
+        adata = adata[self._use_rows_mask, :]
+        adata = adata[:, self._use_cols_mask]
+
+        attrs = [ContinuousVariable.make(str(g)) for g in adata.var_names]
+        meta_df = pd.DataFrame({key: adata.obs[key] for key in adata.obs.keys()})
+
+        X = adata.X.toarray() if sp.issparse(adata.X) else adata.X
+
         return attrs, X, meta_df, meta_df.index
 
 
diff --git a/setup.py b/setup.py
index bc7ec6cb..bef08ccd 100644
--- a/setup.py
+++ b/setup.py
@@ -35,6 +35,7 @@ def include_documentation(local_dir, install_dir):
             'fastdtw==0.3.2',
             'pandas>=0.23',
             'loompy>=2.0.10',
+            'anndata>=0.6.21',
             'numpy',
             'scikit-learn',
         ],