diff --git a/CHANGELOG.md b/CHANGELOG.md index c407ee70..5cc11e4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -Future version +v1.14.0 - Added support for bigBed files diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index 8da7db3f..d9395e05 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -4,10 +4,11 @@ What was changed in this pull request? Why is it necessary? -Fixes #___ +Fixes #\_\_\_ ## Checklist - [ ] Unit tests added or updated - [ ] Documentation added or updated - [ ] Updated CHANGELOG.md +- [ ] Run black diff --git a/fragments/app.py b/fragments/app.py index 764e6ea3..bbdee9bd 100644 --- a/fragments/app.py +++ b/fragments/app.py @@ -4,4 +4,4 @@ class FragmentsConfig(AppConfig): - name = 'fragments' + name = "fragments" diff --git a/fragments/drf_disable_csrf.py b/fragments/drf_disable_csrf.py index 662a80e1..d00bd79e 100644 --- a/fragments/drf_disable_csrf.py +++ b/fragments/drf_disable_csrf.py @@ -2,6 +2,5 @@ class CsrfExemptSessionAuthentication(SessionAuthentication): - def enforce_csrf(self, request): return # To not perform the csrf check previously happening diff --git a/fragments/exceptions.py b/fragments/exceptions.py index 7997775d..f913a7d6 100755 --- a/fragments/exceptions.py +++ b/fragments/exceptions.py @@ -7,10 +7,11 @@ class CoolerFileBroken(APIException): status_code = 500 - default_detail = 'The cooler file is broken.' - default_code = 'cooler_file_broken' + default_detail = "The cooler file is broken." + default_code = "cooler_file_broken" + class SnippetTooLarge(APIException): status_code = 400 - default_detail = 'The requested snippet is too large' - default_code = 'snippet_too_large' + default_detail = "The requested snippet is too large" + default_code = "snippet_too_large" diff --git a/fragments/migrations/0001_initial.py b/fragments/migrations/0001_initial.py index 96e4ed07..121ffd22 100644 --- a/fragments/migrations/0001_initial.py +++ b/fragments/migrations/0001_initial.py @@ -10,32 +10,53 @@ class Migration(migrations.Migration): initial = True - dependencies = [ - ] + dependencies = [] operations = [ migrations.CreateModel( - name='ChromInfo', + name="ChromInfo", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created', models.DateTimeField(auto_now_add=True)), - ('uuid', models.CharField(default=slugid.slugid.nice, max_length=100, unique=True)), - ('datafile', models.TextField()), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ( + "uuid", + models.CharField( + default=slugid.slugid.nice, max_length=100, unique=True + ), + ), + ("datafile", models.TextField()), ], - options={ - 'ordering': ('created',), - }, + options={"ordering": ("created",)}, ), migrations.CreateModel( - name='ChromSizes', + name="ChromSizes", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created', models.DateTimeField(auto_now_add=True)), - ('uuid', models.CharField(default=slugid.slugid.nice, max_length=100, unique=True)), - ('datafile', models.TextField()), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ( + "uuid", + models.CharField( + default=slugid.slugid.nice, max_length=100, unique=True + ), + ), + ("datafile", models.TextField()), ], - options={ - 'ordering': ('created',), - }, + options={"ordering": ("created",)}, ), ] diff --git a/fragments/migrations/0002_auto_20170406_2322.py b/fragments/migrations/0002_auto_20170406_2322.py index b91d7fab..085eef97 100644 --- a/fragments/migrations/0002_auto_20170406_2322.py +++ b/fragments/migrations/0002_auto_20170406_2322.py @@ -7,15 +7,9 @@ class Migration(migrations.Migration): - dependencies = [ - ('fragments', '0001_initial'), - ] + dependencies = [("fragments", "0001_initial")] operations = [ - migrations.DeleteModel( - name='ChromInfo', - ), - migrations.DeleteModel( - name='ChromSizes', - ), + migrations.DeleteModel(name="ChromInfo"), + migrations.DeleteModel(name="ChromSizes"), ] diff --git a/fragments/tests.py b/fragments/tests.py index 9ebd428a..0ece525c 100644 --- a/fragments/tests.py +++ b/fragments/tests.py @@ -10,35 +10,25 @@ class FragmentsTest(dt.TestCase): def setUp(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") upload_file = open( - ( - 'data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb' - '.multires.cool' - ), - 'rb' + ("data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb" ".multires.cool"), + "rb", ) tm.Tileset.objects.create( - datafile=dcfu.SimpleUploadedFile( - upload_file.name, upload_file.read() - ), - filetype='cooler', - uuid='cool-v1', - owner=self.user1 + datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), + filetype="cooler", + uuid="cool-v1", + owner=self.user1, ) upload_file = open( - 'data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.mcoolv2', - 'rb' + "data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.mcoolv2", "rb" ) tm.Tileset.objects.create( - datafile=dcfu.SimpleUploadedFile( - upload_file.name, upload_file.read() - ), - filetype='cooler', - uuid='cool-v2', - owner=self.user1 + datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), + filetype="cooler", + uuid="cool-v2", + owner=self.user1, ) def test_get_fragments(self): @@ -47,30 +37,35 @@ def test_get_fragments(self): data = { "loci": [ [ - "chr1", 1000000000, 2000000000, - "1", 1000000000, 2000000000, f"cool-v{version}", - zoom_res + "chr1", + 1000000000, + 2000000000, + "1", + 1000000000, + 2000000000, + f"cool-v{version}", + zoom_res, ] ] } response = self.client.post( - '/api/v1/fragments_by_loci/?precision=2&dims=22', + "/api/v1/fragments_by_loci/?precision=2&dims=22", json.dumps(data), - content_type="application/json" + content_type="application/json", ) - ret = json.loads(str(response.content, encoding='utf8')) + ret = json.loads(str(response.content, encoding="utf8")) self.assertEqual(response.status_code, 200) - self.assertTrue('fragments' in ret) + self.assertTrue("fragments" in ret) - self.assertEqual(len(ret['fragments']), 1) + self.assertEqual(len(ret["fragments"]), 1) - self.assertEqual(len(ret['fragments'][0]), 22) + self.assertEqual(len(ret["fragments"][0]), 22) - self.assertEqual(len(ret['fragments'][0][0]), 22) + self.assertEqual(len(ret["fragments"][0][0]), 22) def test_string_request_body(self): data = ( @@ -79,105 +74,102 @@ def test_string_request_body(self): ) response = self.client.post( - '/api/v1/fragments_by_loci/?precision=2&dims=22', + "/api/v1/fragments_by_loci/?precision=2&dims=22", json.dumps(data), - content_type="application/json" + content_type="application/json", ) - ret = json.loads(str(response.content, encoding='utf8')) + ret = json.loads(str(response.content, encoding="utf8")) self.assertEqual(response.status_code, 400) - self.assertTrue('error' in ret) - self.assertTrue('error_message' in ret) + self.assertTrue("error" in ret) + self.assertTrue("error_message" in ret) def test_too_large_request(self): for version in [1, 2]: data = [ [ - "1", 1000000000, 2000000000, - "1", 1000000000, 2000000000, - f"cool-v{version}", 0 + "1", + 1000000000, + 2000000000, + "1", + 1000000000, + 2000000000, + f"cool-v{version}", + 0, ] ] response = self.client.post( - '/api/v1/fragments_by_loci/?dims=1025', + "/api/v1/fragments_by_loci/?dims=1025", json.dumps(data), - content_type="application/json" + content_type="application/json", ) - ret = json.loads(str(response.content, encoding='utf8')) + ret = json.loads(str(response.content, encoding="utf8")) self.assertEqual(response.status_code, 400) - self.assertTrue('error' in ret) - self.assertTrue('error_message' in ret) + self.assertTrue("error" in ret) + self.assertTrue("error_message" in ret) def test_both_body_data_types(self): for version in [1, 2]: loci = [ [ - "chr1", 1000000000, 2000000000, - "1", 1000000000, 2000000000, - f"cool-v{version}", 0 + "chr1", + 1000000000, + 2000000000, + "1", + 1000000000, + 2000000000, + f"cool-v{version}", + 0, ] ] - obj = { - "loci": loci - } + obj = {"loci": loci} response = self.client.post( - '/api/v1/fragments_by_loci/?precision=2&dims=22', + "/api/v1/fragments_by_loci/?precision=2&dims=22", json.dumps(obj), - content_type="application/json" + content_type="application/json", ) - ret = json.loads(str(response.content, encoding='utf8')) + ret = json.loads(str(response.content, encoding="utf8")) - mat1 = np.array(ret['fragments'][0], float) + mat1 = np.array(ret["fragments"][0], float) response = self.client.post( - '/api/v1/fragments_by_loci/?precision=2&dims=22', + "/api/v1/fragments_by_loci/?precision=2&dims=22", json.dumps(loci), - content_type="application/json" + content_type="application/json", ) - ret = json.loads(str(response.content, encoding='utf8')) + ret = json.loads(str(response.content, encoding="utf8")) - mat2 = np.array(ret['fragments'][0], float) + mat2 = np.array(ret["fragments"][0], float) self.assertTrue(np.array_equal(mat1, mat2)) def test_negative_start_fragments(self): for version in [1, 2]: - data = [ - [ - "1", - 0, - 1, - "2", - 0, - 1, - f"cool-v{version}", - 20 - ] - ] + data = [["1", 0, 1, "2", 0, 1, f"cool-v{version}", 20]] dims = 60 dims_h = (dims // 2) - 1 response = self.client.post( - '/api/v1/fragments_by_loci/' - '?precision=2&dims={}&no-balance=1'.format(dims), + "/api/v1/fragments_by_loci/" + "?precision=2&dims={}&no-balance=1".format(dims), json.dumps(data), - content_type="application/json" + content_type="application/json", ) self.assertEqual(response.status_code, 200) - ret = json.loads(str(response.content, encoding='utf8')) + ret = json.loads(str(response.content, encoding="utf8")) - self.assertTrue('fragments' in ret) + self.assertTrue("fragments" in ret) - mat = np.array(ret['fragments'][0], float) + mat = np.array(ret["fragments"][0], float) # Upper half should be empty self.assertTrue(np.sum(mat[0:dims_h]) == 0) @@ -189,92 +181,66 @@ def test_domains_by_loci(self): for version in [1, 2]: data = { "loci": [ - [ - "chr1", - 0, - 2000000000, - "1", - 0, - 2000000000, - f"cool-v{version}", - 0 - ] + ["chr1", 0, 2000000000, "1", 0, 2000000000, f"cool-v{version}", 0] ] } response = self.client.post( - '/api/v1/fragments_by_loci/?precision=2&dims=44', + "/api/v1/fragments_by_loci/?precision=2&dims=44", json.dumps(data), - content_type="application/json" + content_type="application/json", ) self.assertEqual(response.status_code, 200) - ret = json.loads(str(response.content, encoding='utf8')) + ret = json.loads(str(response.content, encoding="utf8")) - self.assertTrue('fragments' in ret) + self.assertTrue("fragments" in ret) - self.assertEqual(len(ret['fragments']), 1) + self.assertEqual(len(ret["fragments"]), 1) - self.assertEqual(len(ret['fragments'][0]), 44) + self.assertEqual(len(ret["fragments"][0]), 44) - self.assertEqual(len(ret['fragments'][0][0]), 44) + self.assertEqual(len(ret["fragments"][0][0]), 44) def test_domains_normalizing(self): for version in [1, 2]: - data = [ - [ - "chr2", - 0, - 500000000, - "2", - 0, - 500000000, - f"cool-v{version}", - 0 - ] - ] + data = [["chr2", 0, 500000000, "2", 0, 500000000, f"cool-v{version}", 0]] params = { - 'dims': 60, - 'precision': 3, - 'padding': 2, - 'ignore-diags': 2, - 'percentile': 50 + "dims": 60, + "precision": 3, + "padding": 2, + "ignore-diags": 2, + "percentile": 50, } response = self.client.post( - '/api/v1/fragments_by_loci/?{}'.format(urlencode(params)), + "/api/v1/fragments_by_loci/?{}".format(urlencode(params)), json.dumps(data), - content_type='application/json' + content_type="application/json", ) self.assertEqual(response.status_code, 200) - ret = json.loads(str(response.content, encoding='utf8')) + ret = json.loads(str(response.content, encoding="utf8")) - self.assertTrue('fragments' in ret) + self.assertTrue("fragments" in ret) - mat = np.array(ret['fragments'][0], float) + mat = np.array(ret["fragments"][0], float) # Make sure matrix is not empty self.assertTrue(np.sum(mat) > 0) # Check that the diagonal is 1 (it's being ignored for normalizing # the data but set to 1 to visually make more sense) - diag = np.diag_indices(params['dims']) - self.assertEqual(np.sum(mat[diag]), params['dims']) + diag = np.diag_indices(params["dims"]) + self.assertEqual(np.sum(mat[diag]), params["dims"]) self.assertEqual( - np.sum( - mat[((diag[0] - 1)[1:], diag[1][1:])] - ), - params['dims'] - 1 + np.sum(mat[((diag[0] - 1)[1:], diag[1][1:])]), params["dims"] - 1 ) self.assertEqual( - np.sum( - mat[((diag[0] + 1)[:-1], diag[1][:-1])] - ), - params['dims'] - 1 + np.sum(mat[((diag[0] + 1)[:-1], diag[1][:-1])]), params["dims"] - 1 ) # Check precision of matrix @@ -286,50 +252,50 @@ def test_domains_normalizing(self): # Get two more un-normalized matrices params1 = { - 'dims': 60, - 'precision': 3, - 'padding': 2, - 'ignore-diags': 2, - 'percentile': 50.0, - 'no-normalize': True + "dims": 60, + "precision": 3, + "padding": 2, + "ignore-diags": 2, + "percentile": 50.0, + "no-normalize": True, } response = self.client.post( - '/api/v1/fragments_by_loci/?{}'.format(urlencode(params1)), + "/api/v1/fragments_by_loci/?{}".format(urlencode(params1)), json.dumps(data), - content_type='application/json' + content_type="application/json", ) self.assertEqual(response.status_code, 200) - ret = json.loads(str(response.content, encoding='utf8')) + ret = json.loads(str(response.content, encoding="utf8")) - self.assertTrue('fragments' in ret) + self.assertTrue("fragments" in ret) - mat1 = np.array(ret['fragments'][0], float) + mat1 = np.array(ret["fragments"][0], float) params2 = { - 'dims': 60, - 'precision': 3, - 'padding': 2, - 'ignore-diags': 2, - 'percentile': 100.0, - 'no-normalize': True + "dims": 60, + "precision": 3, + "padding": 2, + "ignore-diags": 2, + "percentile": 100.0, + "no-normalize": True, } response = self.client.post( - '/api/v1/fragments_by_loci/?{}'.format(urlencode(params2)), + "/api/v1/fragments_by_loci/?{}".format(urlencode(params2)), json.dumps(data), - content_type='application/json' + content_type="application/json", ) self.assertEqual(response.status_code, 200) - ret = json.loads(str(response.content, encoding='utf8')) + ret = json.loads(str(response.content, encoding="utf8")) - self.assertTrue('fragments' in ret) + self.assertTrue("fragments" in ret) - mat2 = np.array(ret['fragments'][0], float) + mat2 = np.array(ret["fragments"][0], float) # Make sure matrix is not empty self.assertTrue(np.sum(mat2) > 0) @@ -338,9 +304,9 @@ def test_domains_normalizing(self): self.assertTrue(max2 > max1) - percentile = np.percentile(mat2, params['percentile']) + percentile = np.percentile(mat2, params["percentile"]) self.assertEqual( np.rint(max1 * 10000000) / 10000000, - np.rint(percentile * 10000000) / 10000000 + np.rint(percentile * 10000000) / 10000000, ) diff --git a/fragments/urls.py b/fragments/urls.py index e01d4d15..d1381264 100755 --- a/fragments/urls.py +++ b/fragments/urls.py @@ -5,7 +5,7 @@ # The API URLs are now determined automatically by the router. # Additionally, we include the login URLs for the browsable API. urlpatterns = [ - url(r'^fragments_by_loci/$', views.fragments_by_loci), - url(r'^fragments_by_chr/$', views.fragments_by_chr), - url(r'^loci/$', views.loci), + url(r"^fragments_by_loci/$", views.fragments_by_loci), + url(r"^fragments_by_chr/$", views.fragments_by_chr), + url(r"^loci/$", views.loci), ] diff --git a/fragments/utils.py b/fragments/utils.py index 1e6c1157..06f280b5 100644 --- a/fragments/utils.py +++ b/fragments/utils.py @@ -1,6 +1,4 @@ -from __future__ import ( - absolute_import, division, print_function, unicode_literals -) +from __future__ import absolute_import, division, print_function, unicode_literals import cooler import h5py @@ -38,6 +36,7 @@ # Methods + def grey_to_rgb(arr, to_rgba=False): if to_rgba: rgb = np.zeros(arr.shape + (4,)) @@ -46,8 +45,8 @@ def grey_to_rgb(arr, to_rgba=False): rgb = np.zeros(arr.shape + (3,)) rgb[:, :, 0] = 255 - arr * 255 - rgb[:, :, 1] = rgb[:,:,0] - rgb[:, :, 2] = rgb[:,:,0] + rgb[:, :, 1] = rgb[:, :, 0] + rgb[:, :, 2] = rgb[:, :, 0] return rgb @@ -55,16 +54,16 @@ def grey_to_rgb(arr, to_rgba=False): def blob_to_zip(blobs, to_resp=False): b = BytesIO() - zf = ZipFile(b, 'w') + zf = ZipFile(b, "w") for blob in blobs: - zf.writestr(blob['name'], blob['bytes']) + zf.writestr(blob["name"], blob["bytes"]) zf.close() if to_resp: - resp = HttpResponse(b.getvalue(), content_type='application/zip') - resp['Content-Disposition'] = 'attachment; filename=snippets.zip' + resp = HttpResponse(b.getvalue(), content_type="application/zip") + resp["Content-Disposition"] = "attachment; filename=snippets.zip" return resp @@ -76,27 +75,24 @@ def np_to_png(arr, comp=5): # Add alpha values if arr.shape[2] == 3: - out = np.ones( - (sz[0], sz[1], sz[2] + 1) - ) + out = np.ones((sz[0], sz[1], sz[2] + 1)) out[:, :, 3] = 255 out[:, :, 0:3] = arr else: out = arr return write_png( - np.flipud(out).astype('uint8').flatten('C').tobytes(), - sz[1], - sz[0], - comp + np.flipud(out).astype("uint8").flatten("C").tobytes(), sz[1], sz[0], comp ) def png_pack(png_tag, data): chunk_head = png_tag + data - return (struct.pack("!I", len(data)) + - chunk_head + - struct.pack("!I", 0xFFFFFFFF & zlib.crc32(chunk_head))) + return ( + struct.pack("!I", len(data)) + + chunk_head + + struct.pack("!I", 0xFFFFFFFF & zlib.crc32(chunk_head)) + ) def write_png(buf, width, height, comp=9): @@ -106,16 +102,19 @@ def write_png(buf, width, height, comp=9): # reverse the vertical line order and add null bytes at the start width_byte_4 = width * 4 - raw_data = b''.join( - b'\x00' + buf[span:span + width_byte_4] - for span in np.arange((height - 1) * width_byte_4, -1, - width_byte_4) + raw_data = b"".join( + b"\x00" + buf[span : span + width_byte_4] + for span in np.arange((height - 1) * width_byte_4, -1, -width_byte_4) ) - return b''.join([ - b'\x89PNG\r\n\x1a\n', - png_pack(b'IHDR', struct.pack("!2I5B", width, height, 8, 6, 0, 0, 0)), - png_pack(b'IDAT', zlib.compress(raw_data, comp)), - png_pack(b'IEND', b'')]) + return b"".join( + [ + b"\x89PNG\r\n\x1a\n", + png_pack(b"IHDR", struct.pack("!2I5B", width, height, 8, 6, 0, 0, 0)), + png_pack(b"IDAT", zlib.compress(raw_data, comp)), + png_pack(b"IEND", b""), + ] + ) def get_params(request, param_def): @@ -139,25 +138,20 @@ def get_params(request, param_def): dict -- Dictionary of parameter name<>values pairs """ p = {} - dtype = { - 'int': int, - 'float': float, - 'bool': bool, - 'str': str, - } + dtype = {"int": int, "float": float, "bool": bool, "str": str} for key in param_def.keys(): - p[key] = request.GET.get(param_def[key]['short']) + p[key] = request.GET.get(param_def[key]["short"]) p[key] = p[key] if p[key] else request.GET.get(key) - p[key] = p[key] if p[key] else param_def[key]['default'] - p[key] = dtype[param_def[key]['dtype']](p[key]) + p[key] = p[key] if p[key] else param_def[key]["default"] + p[key] = dtype[param_def[key]["dtype"]](p[key]) return p def get_chrom_names_cumul_len(c): - ''' + """ Get the chromosome names and cumulative lengths: Args: @@ -167,7 +161,7 @@ def get_chrom_names_cumul_len(c): Return: (names, sizes, lengths) -> (list(string), dict, np.array(int)) - ''' + """ chrom_sizes = {} chrom_cum_lengths = [0] @@ -190,43 +184,41 @@ def get_chrom_names_cumul_len(c): def get_intra_chr_loops_from_looplist(loop_list, chr=0): - loops = pd.DataFrame.from_csv(loop_list, sep='\t', header=0) + loops = pd.DataFrame.from_csv(loop_list, sep="\t", header=0) s_chr = str(chr) if chr > 0: - loops = loops[loops['chr2'] == s_chr].loc[s_chr] + loops = loops[loops["chr2"] == s_chr].loc[s_chr] chrs = np.zeros((loops.shape[0], 2), dtype=object) - chrs[:, 0] = loops['chr2'] - chrs[:, 1] = loops['chr2'] + chrs[:, 0] = loops["chr2"] + chrs[:, 1] = loops["chr2"] return (loops.as_matrix()[:, [0, 1, 3, 4]], chrs) def rel_2_abs_loci(loci, chr_info): - ''' + """ chr_info[0] = chromosome names chr_info[1] = chromosome lengths chr_info[2] = cumulative lengths chr_info[3] = chromosome ids - ''' + """ + def absolutize(chr, x, y): chrom = str(chr) - if not chrom.startswith('chr'): - chrom = 'chr{}'.format(chrom) + if not chrom.startswith("chr"): + chrom = "chr{}".format(chrom) offset = chr_info[2][chr_info[3][chrom]] return (offset + x, offset + y) def absolutize_tuple(tuple): - return ( - absolutize(*tuple[0:3]) + - absolutize(*tuple[3:6]) - ) + return absolutize(*tuple[0:3]) + absolutize(*tuple[3:6]) return list(map(absolutize_tuple, loci)) @@ -239,19 +231,20 @@ def get_cooler(f, zoomout_level=None): # In this case `zoomout_level` is the resolution # See fragments/views.py line 431 - resolutions = [int(x) for x in f['resolutions'].keys()] + resolutions = [int(x) for x in f["resolutions"].keys()] resolution = min(resolutions) if zoomout_level is None else zoomout_level # Get the closest zoomlevel - resolution = resolutions[np.argsort([abs(r - resolution) for r in resolutions])[0]] + resolution = resolutions[ + np.argsort([abs(r - resolution) for r in resolutions])[0] + ] - return cooler.Cooler(f['resolutions/{}'.format(resolution)]) + return cooler.Cooler(f["resolutions/{}".format(resolution)]) except: # We're not logging this exception as the user might just try to open # a cooler v1 file pass - try: # v1 zoomout_level = 0 if zoomout_level is None else zoomout_level @@ -262,10 +255,10 @@ def get_cooler(f, zoomout_level=None): zoom_level = max_zoom - max(zoomout_level, 0) - if (zoom_level >= min_zoom and zoom_level <= max_zoom): + if zoom_level >= min_zoom and zoom_level <= max_zoom: c = cooler.Cooler(f[str(zoom_level)]) else: - c = cooler.Cooler(f['0']) + c = cooler.Cooler(f["0"]) return c except Exception as e: @@ -291,11 +284,11 @@ def get_frag_by_loc_from_cool( no_normalize=False, aggregate=False, ): - with h5py.File(cooler_file, 'r') as f: + with h5py.File(cooler_file, "r") as f: c = get_cooler(f, zoomout_level) # Calculate the offsets once - resolution = c.info['bin-size'] + resolution = c.info["bin-size"] chromsizes = np.ceil(c.chromsizes / resolution).astype(int) offsets = np.cumsum(chromsizes) - chromsizes @@ -310,7 +303,7 @@ def get_frag_by_loc_from_cool( percentile=percentile, ignore_diags=ignore_diags, no_normalize=no_normalize, - aggregate=aggregate + aggregate=aggregate, ) return fragments @@ -368,12 +361,12 @@ def get_scale_frags_to_same_size(frags, loci_ids, out_size=-1, no_cache=False): out = np.zeros([len(frags), dim_x, dim_y]) for i, frag in enumerate(frags): - id = loci_ids[i] + '.' + '.'.join(map(str, out.shape[1:])) + id = loci_ids[i] + "." + ".".join(map(str, out.shape[1:])) if not no_cache: frag_ds = None try: - frag_ds = np.load(BytesIO(rdb.get('im_snip_ds_%s' % id))) + frag_ds = np.load(BytesIO(rdb.get("im_snip_ds_%s" % id))) if frag_ds is not None: out[i] = frag_ds continue @@ -402,7 +395,7 @@ def get_scale_frags_to_same_size(frags, loci_ids, out_size=-1, no_cache=False): if not no_cache: with BytesIO() as b: np.save(b, frag) - rdb.set('im_snip_ds_%s' % id, b.getvalue(), 60 * 30) + rdb.set("im_snip_ds_%s" % id, b.getvalue(), 60 * 30) out[i] = frag @@ -430,9 +423,7 @@ def get_rep_frags(frags, loci, loci_ids, num_reps=4, no_cache=False): return [frags[i] for i in idx], idx - out, _, _ = get_scale_frags_to_same_size( - frags, loci_ids, 32, no_cache - ) + out, _, _ = get_scale_frags_to_same_size(frags, loci_ids, 32, no_cache) # Get largest frag based on world coords largest_a = 0 @@ -447,9 +438,7 @@ def get_rep_frags(frags, loci, loci_ids, num_reps=4, no_cache=False): # Sum each x,y and c (channel) value up per f (fragment) and take the # sqaure root to get the L2 norm - dist_to_mean = np.sqrt( - np.einsum('fxyc,fxyc->f', diff_mean_frags, diff_mean_frags) - ) + dist_to_mean = np.sqrt(np.einsum("fxyc,fxyc->f", diff_mean_frags, diff_mean_frags)) # Get the fragment closest to the mean # Get the index of the i-th smallest value i=0 == smallest value @@ -461,24 +450,24 @@ def get_rep_frags(frags, loci, loci_ids, num_reps=4, no_cache=False): for i in range(len(dist_to_mean) - 1, -1, -1): farthest_mean_frag_idx = np.argpartition(dist_to_mean, i)[i] if ( - farthest_mean_frag_idx != largest_frag_idx and - farthest_mean_frag_idx != closest_mean_frag_idx + farthest_mean_frag_idx != largest_frag_idx + and farthest_mean_frag_idx != closest_mean_frag_idx ): break # Distance to farthest away frag diff_farthest_frags = out - out[np.argmax(dist_to_mean)] dist_to_farthest = np.sqrt( - np.einsum('fxyc,fxyc->f', diff_farthest_frags, diff_farthest_frags) + np.einsum("fxyc,fxyc->f", diff_farthest_frags, diff_farthest_frags) ) # Get the frag farthest away from the frag farthest away from the mean for i in range(len(dist_to_farthest) - 1, -1, -1): farthest_farthest_frag_idx = np.argpartition(dist_to_farthest, i)[i] if ( - farthest_farthest_frag_idx != largest_frag_idx and - farthest_farthest_frag_idx != closest_mean_frag_idx and - farthest_farthest_frag_idx != farthest_mean_frag_idx + farthest_farthest_frag_idx != largest_frag_idx + and farthest_farthest_frag_idx != closest_mean_frag_idx + and farthest_farthest_frag_idx != farthest_mean_frag_idx ): break @@ -486,25 +475,20 @@ def get_rep_frags(frags, loci, loci_ids, num_reps=4, no_cache=False): frags[largest_frag_idx], frags[closest_mean_frag_idx], frags[farthest_mean_frag_idx], - frags[farthest_farthest_frag_idx] + frags[farthest_farthest_frag_idx], ] idx = [ largest_frag_idx, closest_mean_frag_idx, farthest_mean_frag_idx, - farthest_farthest_frag_idx + farthest_farthest_frag_idx, ] return frags, idx -def aggregate_frags( - frags, - loci_ids, - method='mean', - max_previews=8, -): +def aggregate_frags(frags, loci_ids, method="mean", max_previews=8): """Aggregate multiple fragments into one Arguments: @@ -534,7 +518,7 @@ def aggregate_frags( previews_2d = [] - if method == 'median': + if method == "median": aggregate = np.nanmedian(out, axis=0) if len(frags) > max_previews: for i in range(max_previews): @@ -545,30 +529,26 @@ def aggregate_frags( previews = np.nanmedian(out, axis=1) return aggregate, previews - elif method == 'std': + elif method == "std": aggregate = np.nanstd(out, axis=0) if len(frags) > max_previews: for i in range(max_previews): - previews[i] = np.nanstd( - out[np.where(clusters.labels_ == i)], axis=1 - )[0] + previews[i] = np.nanstd(out[np.where(clusters.labels_ == i)], axis=1)[0] else: previews = np.nanmedian(out, axis=1) return aggregate, previews - elif method == 'var': + elif method == "var": aggregate = np.nanvar(out, axis=0) if len(frags) > max_previews: for i in range(max_previews): - previews[i] = np.nanvar( - out[np.where(clusters.labels_ == i)], axis=1 - )[0] + previews[i] = np.nanvar(out[np.where(clusters.labels_ == i)], axis=1)[0] else: previews = np.nanmedian(out, axis=1) return aggregate, previews - elif method != 'mean': - print('Unknown aggregation method: {}'.format(method)) + elif method != "mean": + print("Unknown aggregation method: {}".format(method)) aggregate = np.nanmean(out, axis=0) if max_previews > 0: @@ -578,9 +558,9 @@ def aggregate_frags( previews[i] = np.nanmean( out[np.where(clusters.labels_ == i)[0]], axis=1 )[0] - previews_2d.append(np.nanmean( - out[np.where(clusters.labels_ == i)], axis=0 - )) + previews_2d.append( + np.nanmean(out[np.where(clusters.labels_ == i)], axis=0) + ) else: previews = np.nanmedian(out, axis=1) previews_2d = frags @@ -601,14 +581,13 @@ def get_frag_from_image_tiles( from_x, to_x, from_y, - to_y + to_y, ): im = ( tiles[0] if len(tiles) == 1 else Image.new( - 'RGB', - (tile_size * len(tiles_x_range), tile_size * len(tiles_y_range)) + "RGB", (tile_size * len(tiles_x_range), tile_size * len(tiles_y_range)) ) ) @@ -631,12 +610,7 @@ def get_frag_from_image_tiles( def get_frag_by_loc_from_imtiles( - imtiles_file, - loci, - zoom_level=0, - padding=0, - tile_size=256, - no_cache=False + imtiles_file, loci, zoom_level=0, padding=0, tile_size=256, no_cache=False ): db = None div = 1 @@ -653,7 +627,7 @@ def get_frag_by_loc_from_imtiles( if not no_cache: im_snip = None try: - im_snip = np.load(BytesIO(rdb.get('im_snip_%s' % id))) + im_snip = np.load(BytesIO(rdb.get("im_snip_%s" % id))) if im_snip is not None: ims.append(im_snip) continue @@ -662,7 +636,7 @@ def get_frag_by_loc_from_imtiles( if not got_info: db = sqlite3.connect(imtiles_file) - info = db.execute('SELECT * FROM tileset_info').fetchone() + info = db.execute("SELECT * FROM tileset_info").fetchone() max_zoom = info[6] max_width = info[8] @@ -702,10 +676,16 @@ def get_frag_by_loc_from_imtiles( tiles = [] for y in tiles_y_range: for x in tiles_x_range: - tiles.append(Image.open(BytesIO(db.execute( - 'SELECT image FROM tiles WHERE z=? AND y=? AND x=?', - (zoom_level, y, x) - ).fetchone()[0]))) + tiles.append( + Image.open( + BytesIO( + db.execute( + "SELECT image FROM tiles WHERE z=? AND y=? AND x=?", + (zoom_level, y, x), + ).fetchone()[0] + ) + ) + ) im_snip = get_frag_from_image_tiles( tiles, @@ -717,14 +697,14 @@ def get_frag_by_loc_from_imtiles( start1, end1, start2, - end2 + end2, ) # Cache for 30 min if not no_cache: with BytesIO() as b: np.save(b, im_snip) - rdb.set('im_snip_%s' % id, b.getvalue(), 60 * 30) + rdb.set("im_snip_%s" % id, b.getvalue(), 60 * 30) ims.append(im_snip) @@ -735,21 +715,16 @@ def get_frag_by_loc_from_imtiles( def get_frag_by_loc_from_osm( - imtiles_file, - loci, - zoom_level=0, - padding=0, - tile_size=256, - no_cache=False + imtiles_file, loci, zoom_level=0, padding=0, tile_size=256, no_cache=False ): width = 360 height = 180 ims = [] - prefixes = ['a', 'b', 'c'] + prefixes = ["a", "b", "c"] prefix_idx = math.floor(random() * len(prefixes)) - osm_src = 'http://{}.tile.openstreetmap.org'.format(prefixes[prefix_idx]) + osm_src = "http://{}.tile.openstreetmap.org".format(prefixes[prefix_idx]) s = CacheControl(requests.Session()) @@ -759,7 +734,7 @@ def get_frag_by_loc_from_osm( if not no_cache: osm_snip = None try: - osm_snip = np.load(BytesIO(rdb.get('osm_snip_%s' % id))) + osm_snip = np.load(BytesIO(rdb.get("osm_snip_%s" % id))) if osm_snip is not None: ims.append(osm_snip) continue @@ -772,23 +747,14 @@ def get_frag_by_loc_from_osm( end_lat = locus[3] if not is_within( - start_lng + 180, - end_lng + 180, - end_lat + 90, - start_lat + 90, - width, - height + start_lng + 180, end_lng + 180, end_lat + 90, start_lat + 90, width, height ): ims.append(None) continue # Get tile ids - start1, start2 = get_tile_pos_from_lng_lat( - start_lng, start_lat, zoom_level - ) - end1, end2 = get_tile_pos_from_lng_lat( - end_lng, end_lat, zoom_level - ) + start1, start2 = get_tile_pos_from_lng_lat(start_lng, start_lat, zoom_level) + end1, end2 = get_tile_pos_from_lng_lat(end_lng, end_lat, zoom_level) xPad = padding * (end1 - start1) yPad = padding * (start2 - end2) @@ -821,17 +787,12 @@ def get_frag_by_loc_from_osm( tiles = [] for y in tiles_y_range: for x in tiles_x_range: - src = ( - '{}/{}/{}/{}.png' - .format(osm_src, zoom_level, x, y) - ) + src = "{}/{}/{}/{}.png".format(osm_src, zoom_level, x, y) r = s.get(src) if r.status_code == 200: - tiles.append(Image.open( - BytesIO(r.content) - ).convert('RGB')) + tiles.append(Image.open(BytesIO(r.content)).convert("RGB")) else: tiles.append(None) @@ -845,13 +806,13 @@ def get_frag_by_loc_from_osm( start1, end1, start2, - end2 + end2, ) if not no_cache: with BytesIO() as b: np.save(b, osm_snip) - rdb.set('osm_snip_%s' % id, b.getvalue(), 60 * 30) + rdb.set("osm_snip_%s" % id, b.getvalue(), 60 * 30) ims.append(osm_snip) @@ -863,26 +824,26 @@ def is_within(start1, end1, start2, end2, width, height): def calc_measure_dtd(matrix, locus): - ''' + """ Calculate the distance to the diagonal - ''' - return np.abs(locus['end1'] - locus['start2']) + """ + return np.abs(locus["end1"] - locus["start2"]) def calc_measure_size(matrix, locus, bin_size=1): - ''' + """ Calculate the size of the snippet - ''' + """ return ( - np.abs(locus['start1'] - locus['end1']) * - np.abs(locus['start2'] - locus['end2']) + np.abs(locus["start1"] - locus["end1"]) + * np.abs(locus["start2"] - locus["end2"]) ) / bin_size def calc_measure_noise(matrix): - ''' + """ Estimate the noise level of the input matrix using the standard deviation - ''' + """ low_quality_bins = np.where(matrix == -1) # Assign 0 for now to avoid influencing the standard deviation @@ -918,10 +879,9 @@ def calc_measure_sharpness(matrix): for i in range(dim): for j in range(dim): - var += ( - ((i - (middle + i // m)) ** 2 + (j - middle + i // m) ** 2) * - matrix[i, j] - ) + var += ((i - (middle + i // m)) ** 2 + (j - middle + i // m) ** 2) * matrix[ + i, j + ] # Ressign -1 to low quality bins matrix[low_quality_bins] = -1 @@ -930,7 +890,7 @@ def calc_measure_sharpness(matrix): def get_bin_size(cooler_file, zoomout_level=-1): - with h5py.File(cooler_file, 'r') as f: + with h5py.File(cooler_file, "r") as f: c = get_cooler(f, zoomout_level) return c.util.get_binsize() @@ -952,24 +912,26 @@ def collect_frags( percentile=100.0, ignore_diags=0, no_normalize=False, - aggregate=False + aggregate=False, ): fragments = [] for locus in loci: last_loc = len(locus) - 2 - fragments.append(get_frag( - c, - resolution, - offsets, - *locus[:6], - width=locus[last_loc] if locus[last_loc] else dim, - padding=padding, - balanced=balanced, - percentile=percentile, - ignore_diags=ignore_diags, - no_normalize=no_normalize - )) + fragments.append( + get_frag( + c, + resolution, + offsets, + *locus[:6], + width=locus[last_loc] if locus[last_loc] else dim, + padding=padding, + balanced=balanced, + percentile=percentile, + ignore_diags=ignore_diags, + no_normalize=no_normalize + ) + ) return fragments @@ -993,7 +955,7 @@ def get_chroms(abs_pos, chr_info=None, cooler_file=None, zoomout_level=-1): chroms = np.zeros((abs_pos.shape[0], 2), dtype=object) if chr_info is None: - with h5py.File(cooler_file, 'r') as f: + with h5py.File(cooler_file, "r") as f: c = get_cooler(f, zoomout_level) chr_info = get_chrom_names_cumul_len(c) @@ -1010,22 +972,26 @@ def rel_loci_2_obj(loci_rel_chroms): i = 0 for locus in loci_rel_chroms: - loci.append({ - 'chrom1': loci_rel_chroms[i, 0], - 'start1': loci_rel_chroms[i, 1], - 'end1': loci_rel_chroms[i, 2], - 'strand1': ( - 'coding' if loci_rel_chroms[i, 1] < loci_rel_chroms[i, 2] else - 'noncoding' - ), - 'chrom2': loci_rel_chroms[i, 3], - 'start2': loci_rel_chroms[i, 4], - 'end2': loci_rel_chroms[i, 5], - 'strand2': ( - 'coding' if loci_rel_chroms[i, 1] < loci_rel_chroms[i, 2] else - 'noncoding' - ) - }) + loci.append( + { + "chrom1": loci_rel_chroms[i, 0], + "start1": loci_rel_chroms[i, 1], + "end1": loci_rel_chroms[i, 2], + "strand1": ( + "coding" + if loci_rel_chroms[i, 1] < loci_rel_chroms[i, 2] + else "noncoding" + ), + "chrom2": loci_rel_chroms[i, 3], + "start2": loci_rel_chroms[i, 4], + "end2": loci_rel_chroms[i, 5], + "strand2": ( + "coding" + if loci_rel_chroms[i, 1] < loci_rel_chroms[i, 2] + else "noncoding" + ), + } + ) i += 1 return loci @@ -1035,7 +1001,7 @@ def abs_coord_2_bin(c, pos, chr_info): try: chr_id = np.flatnonzero(chr_info[2] > pos)[0] - 1 except IndexError: - return c.info['nbins'] + return c.info["nbins"] chrom = chr_info[0][chr_id] relPos = pos - chr_info[2][chr_id] @@ -1060,7 +1026,7 @@ def get_frag( balanced: bool = True, percentile: float = 100.0, ignore_diags: int = 0, - no_normalize: bool = False + no_normalize: bool = False, ) -> np.ndarray: """ Retrieves a matrix fragment. @@ -1119,13 +1085,13 @@ def get_frag( offset1 = offsets[chrom1] except KeyError: # One more try before we will fail miserably - offset1 = offsets['chr{}'.format(chrom1)] + offset1 = offsets["chr{}".format(chrom1)] try: offset2 = offsets[chrom2] except KeyError: # One more try before we will fail miserably - offset2 = offsets['chr{}'.format(chrom2)] + offset2 = offsets["chr{}".format(chrom2)] start_bin1 = offset1 + int(round(float(start1) / resolution)) end_bin1 = offset1 + int(round(float(end1) / resolution)) + 1 @@ -1172,8 +1138,10 @@ def get_frag( abs_dim2 = height # Maximum width / height is 512 - if abs_dim1 > hss.SNIPPET_MAT_MAX_DATA_DIM: raise SnippetTooLarge() - if abs_dim2 > hss.SNIPPET_MAT_MAX_DATA_DIM: raise SnippetTooLarge() + if abs_dim1 > hss.SNIPPET_MAT_MAX_DATA_DIM: + raise SnippetTooLarge() + if abs_dim2 > hss.SNIPPET_MAT_MAX_DATA_DIM: + raise SnippetTooLarge() # Finally, adjust to negative values. # Since relative bin IDs are adjusted by the start this will lead to a @@ -1182,31 +1150,31 @@ def get_frag( real_start_bin2 = start_bin2 if start_bin2 >= 0 else 0 # Get the data - data = c.matrix( - as_pixels=True, balance=False, max_chunk=np.inf - )[real_start_bin1:end_bin1, real_start_bin2:end_bin2] + data = c.matrix(as_pixels=True, balance=False, max_chunk=np.inf)[ + real_start_bin1:end_bin1, real_start_bin2:end_bin2 + ] # Annotate pixels for balancing - bins = c.bins(convert_enum=False)[['weight']] + bins = c.bins(convert_enum=False)[["weight"]] data = cooler.annotate(data, bins, replace=False) # Calculate relative bin IDs - rel_bin1 = np.add(data['bin1_id'].values, -start_bin1) - rel_bin2 = np.add(data['bin2_id'].values, -start_bin2) + rel_bin1 = np.add(data["bin1_id"].values, -start_bin1) + rel_bin2 = np.add(data["bin2_id"].values, -start_bin2) # Balance counts if balanced: - values = data['count'].values.astype(np.float32) - values *= data['weight1'].values * data['weight2'].values + values = data["count"].values.astype(np.float32) + values *= data["weight1"].values * data["weight2"].values else: - values = data['count'].values + values = data["count"].values # Get pixel IDs for the upper triangle idx1 = np.add(np.multiply(rel_bin1, abs_dim1), rel_bin2) # Mirror matrix - idx2_1 = np.add(data['bin2_id'].values, -start_bin1) - idx2_2 = np.add(data['bin1_id'].values, -start_bin2) + idx2_1 = np.add(data["bin2_id"].values, -start_bin1) + idx2_2 = np.add(data["bin1_id"].values, -start_bin2) idx2 = np.add(np.multiply(idx2_1, abs_dim1), idx2_2) validBins = np.where((idx2_1 < abs_dim1) & (idx2_2 >= 0)) @@ -1215,11 +1183,9 @@ def get_frag( if ignore_diags > 0: try: diags_start_idx = np.min( - np.where(data['bin1_id'].values == data['bin2_id'].values) - ) - diags_start_row = ( - rel_bin1[diags_start_idx] - rel_bin2[diags_start_idx] + np.where(data["bin1_id"].values == data["bin2_id"].values) ) + diags_start_row = rel_bin1[diags_start_idx] - rel_bin2[diags_start_idx] except ValueError: pass @@ -1243,9 +1209,7 @@ def get_frag( scale_x = width / frag.shape[0] if frag.shape[0] > width or frag.shape[1] > height: scaledFrag = np.zeros((width, height), float) - frag = scaledFrag + zoomArray( - frag, scaledFrag.shape, order=1 - ) + frag = scaledFrag + zoomArray(frag, scaledFrag.shape, order=1) scaled = True # Normalize by minimum @@ -1262,9 +1226,7 @@ def get_frag( idx = np.diag_indices(width) scaled_idx = ( - idx - if scaled_row == 0 - else [idx[0][scaled_row:], idx[0][:-scaled_row]] + idx if scaled_row == 0 else [idx[0][scaled_row:], idx[0][:-scaled_row]] ) for i in range(ignore_diags): @@ -1279,28 +1241,22 @@ def get_frag( off = 0 if dist_to_diag >= 0 else i - scaled_row # Above diagonal - frag[ - ((scaled_idx[0] - i)[off:], (scaled_idx[1])[off:]) - ] = -1 + frag[((scaled_idx[0] - i)[off:], (scaled_idx[1])[off:])] = -1 # Extra cutoff at the bottom right frag[ ( range( - scaled_idx[0][-1] - i, - scaled_idx[0][-1] + 1 + dist_neg, + scaled_idx[0][-1] - i, scaled_idx[0][-1] + 1 + dist_neg ), range( - scaled_idx[1][-1], - scaled_idx[1][-1] + i + 1 + dist_neg - ) + scaled_idx[1][-1], scaled_idx[1][-1] + i + 1 + dist_neg + ), ) ] = -1 # Below diagonal - frag[ - ((scaled_idx[0] + i)[:-i], (scaled_idx[1])[:-i]) - ] = -1 + frag[((scaled_idx[0] + i)[:-i], (scaled_idx[1])[:-i])] = -1 # Save the final selection of ignored cells for fast access # later and set those values to `0` now. @@ -1308,9 +1264,7 @@ def get_frag( frag[ignored_idx] = 0 else: - logger.warn( - 'Ignoring the diagonal only supported for squared features' - ) + logger.warn("Ignoring the diagonal only supported for squared features") # Capp by percentile max_val = np.percentile(frag, percentile) @@ -1331,9 +1285,7 @@ def get_frag( return frag -def zoomArray( - inArray, finalShape, sameSum=False, zoomFunction=zoom, **zoomKwargs -): +def zoomArray(inArray, finalShape, sameSum=False, zoomFunction=zoom, **zoomKwargs): """ Normally, one can use scipy.ndimage.zoom to do array/image rescaling. However, scipy.ndimage.zoom does not coarsegrain images well. It basically @@ -1388,7 +1340,7 @@ def zoomArray( if mult != 1: sh = list(rescaled.shape) assert sh[ind] % mult == 0 - newshape = sh[:ind] + [sh[ind] // mult, mult] + sh[ind + 1:] + newshape = sh[:ind] + [sh[ind] // mult, mult] + sh[ind + 1 :] rescaled.shape = newshape rescaled = np.mean(rescaled, axis=ind + 1) assert rescaled.shape == finalShape diff --git a/fragments/views.py b/fragments/views.py index de1c94ab..ea352d65 100755 --- a/fragments/views.py +++ b/fragments/views.py @@ -6,6 +6,7 @@ import numpy as np import pybase64 from PIL import Image + try: import cPickle as pickle except: @@ -36,7 +37,7 @@ np_to_png, write_png, grey_to_rgb, - blob_to_zip + blob_to_zip, ) from higlass_server.utils import getRdb from fragments.exceptions import SnippetTooLarge @@ -49,119 +50,113 @@ logger = logging.getLogger(__name__) -SUPPORTED_MEASURES = ['distance-to-diagonal', 'noise', 'size', 'sharpness'] +SUPPORTED_MEASURES = ["distance-to-diagonal", "noise", "size", "sharpness"] -SUPPORTED_FILETYPES = ['matrix', 'im-tiles', 'osm-tiles'] +SUPPORTED_FILETYPES = ["matrix", "im-tiles", "osm-tiles"] GET_FRAG_PARAMS = { - 'dims': { - 'short': 'di', - 'dtype': 'int', - 'default': 22, - 'help': 'Global number of dimensions. (Only used for cooler tilesets.)' + "dims": { + "short": "di", + "dtype": "int", + "default": 22, + "help": "Global number of dimensions. (Only used for cooler tilesets.)", }, - 'padding': { - 'short': 'pd', - 'dtype': 'float', - 'default': 0, - 'help': 'Add given percent of the fragment size as padding.' + "padding": { + "short": "pd", + "dtype": "float", + "default": 0, + "help": "Add given percent of the fragment size as padding.", }, - 'no-balance': { - 'short': 'nb', - 'dtype': 'bool', - 'default': False, - 'help': ( - 'Do not balance fragmens if true. (Only used for cooler tilesets.)' - ) + "no-balance": { + "short": "nb", + "dtype": "bool", + "default": False, + "help": ("Do not balance fragmens if true. (Only used for cooler tilesets.)"), }, - 'percentile': { - 'short': 'pe', - 'dtype': 'float', - 'default': 100.0, - 'help': ( - 'Cap values at given percentile. (Only used for cooler tilesets.)' - ) + "percentile": { + "short": "pe", + "dtype": "float", + "default": 100.0, + "help": ("Cap values at given percentile. (Only used for cooler tilesets.)"), }, - 'precision': { - 'short': 'pr', - 'dtype': 'int', - 'default': 0, - 'help': ( - 'Number of decimals of the numerical values. ' - '(Only used for cooler tilesets.)' - ) + "precision": { + "short": "pr", + "dtype": "int", + "default": 0, + "help": ( + "Number of decimals of the numerical values. " + "(Only used for cooler tilesets.)" + ), }, - 'no-cache': { - 'short': 'nc', - 'dtype': 'bool', - 'default': 0, - 'help': 'Do not cache fragments if true. Useful for debugging.' + "no-cache": { + "short": "nc", + "dtype": "bool", + "default": 0, + "help": "Do not cache fragments if true. Useful for debugging.", }, - 'ignore-diags': { - 'short': 'nd', - 'dtype': 'int', - 'default': 0, - 'help': ( - 'Ignore N diagonals, i.e., set them to zero. ' - '(Only used for cooler tilesets.)' - ) + "ignore-diags": { + "short": "nd", + "dtype": "int", + "default": 0, + "help": ( + "Ignore N diagonals, i.e., set them to zero. " + "(Only used for cooler tilesets.)" + ), }, - 'no-normalize': { - 'short': 'nn', - 'dtype': 'bool', - 'default': False, - 'help': ( - 'Do not normalize fragments if true. ' - '(Only used for cooler tilesets.)' - ) + "no-normalize": { + "short": "nn", + "dtype": "bool", + "default": False, + "help": ( + "Do not normalize fragments if true. " "(Only used for cooler tilesets.)" + ), }, - 'aggregate': { - 'short': 'ag', - 'dtype': 'bool', - 'default': False, - 'help': 'Aggregate fragments if true.' + "aggregate": { + "short": "ag", + "dtype": "bool", + "default": False, + "help": "Aggregate fragments if true.", }, - 'aggregation-method': { - 'short': 'am', - 'dtype': 'str', - 'default': 'mean', - 'help': 'Aggregation method: mean, median, std, var.' + "aggregation-method": { + "short": "am", + "dtype": "str", + "default": "mean", + "help": "Aggregation method: mean, median, std, var.", }, - 'max-previews': { - 'short': 'mp', - 'dtype': 'int', - 'default': 0, - 'help': ( - 'Max. number of 1D previews to return. When the number of ' - 'fragments s higher than the previews we cluster the frags by ' - 'k-means.' - ) + "max-previews": { + "short": "mp", + "dtype": "int", + "default": 0, + "help": ( + "Max. number of 1D previews to return. When the number of " + "fragments s higher than the previews we cluster the frags by " + "k-means." + ), }, - 'encoding': { - 'short': 'en', - 'dtype': 'str', - 'default': 'matrix', - 'help': ( - 'Data encoding: matrix, b64, or image. (Image encoding only ' - 'supported when one fragment is to be returned)' - ) + "encoding": { + "short": "en", + "dtype": "str", + "default": "matrix", + "help": ( + "Data encoding: matrix, b64, or image. (Image encoding only " + "supported when one fragment is to be returned)" + ), }, - 'representatives': { - 'short': 'rp', - 'dtype': 'int', - 'default': 0, - 'help': ( - 'Number of representative fragments when requesting multiple ' - 'fragments.' - ) + "representatives": { + "short": "rp", + "dtype": "int", + "default": 0, + "help": ( + "Number of representative fragments when requesting multiple " "fragments." + ), }, } -@api_view(['GET', 'POST']) +@api_view(["GET", "POST"]) @authentication_classes((CsrfExemptSessionAuthentication, BasicAuthentication)) def fragments_by_loci(request): - if request.method == 'GET': + if request.method == "GET": return get_fragments_by_loci_info(request) return get_fragments_by_loci(request) @@ -172,7 +167,7 @@ def get_fragments_by_loci_info(request): def get_fragments_by_loci(request): - ''' + """ Retrieve a list of locations and return the corresponding matrix fragments Args: @@ -182,31 +177,34 @@ def get_fragments_by_loci(request): Return: - ''' + """ if type(request.data) is str: - return JsonResponse({ - 'error': 'Request body needs to be an array or object.', - 'error_message': 'Request body needs to be an array or object.' - }, status=400) + return JsonResponse( + { + "error": "Request body needs to be an array or object.", + "error_message": "Request body needs to be an array or object.", + }, + status=400, + ) try: - loci = request.data.get('loci', []) + loci = request.data.get("loci", []) except AttributeError: loci = request.data except Exception as e: - return JsonResponse({ - 'error': 'Could not read request body.', - 'error_message': str(e) - }, status=400) + return JsonResponse( + {"error": "Could not read request body.", "error_message": str(e)}, + status=400, + ) try: - forced_rep_idx = request.data.get('representativeIndices', None) + forced_rep_idx = request.data.get("representativeIndices", None) except Exception as e: forced_rep_idx = None pass - ''' + """ Loci list must be of type: [cooler] [imtiles] 0: chrom1 start1 @@ -220,23 +218,23 @@ def get_fragments_by_loci(request): 8: dim* *) Optional - ''' + """ params = get_params(request, GET_FRAG_PARAMS) - dims = params['dims'] - padding = params['padding'] - no_balance = params['no-balance'] - percentile = params['percentile'] - precision = params['precision'] - no_cache = params['no-cache'] - ignore_diags = params['ignore-diags'] - no_normalize = params['no-normalize'] - aggregate = params['aggregate'] - aggregation_method = params['aggregation-method'] - max_previews = params['max-previews'] - encoding = params['encoding'] - representatives = params['representatives'] + dims = params["dims"] + padding = params["padding"] + no_balance = params["no-balance"] + percentile = params["percentile"] + precision = params["precision"] + no_cache = params["no-cache"] + ignore_diags = params["ignore-diags"] + no_normalize = params["no-normalize"] + aggregate = params["aggregate"] + aggregation_method = params["aggregation-method"] + max_previews = params["max-previews"] + encoding = params["encoding"] + representatives = params["representatives"] # Check if requesting a snippet from a `.cool` cooler file is_cool = len(loci) and len(loci[0]) > 7 @@ -255,65 +253,66 @@ def get_fragments_by_loci(request): loci_ids = [] try: for locus in loci: - tileset_file = '' + tileset_file = "" if locus[tileset_idx]: if locus[tileset_idx] in ts_cache: - tileset = ts_cache[locus[tileset_idx]]['obj'] - tileset_file = ts_cache[locus[tileset_idx]]['path'] - elif locus[tileset_idx].endswith('.cool'): - tileset_file = path.join('data', locus[tileset_idx]) + tileset = ts_cache[locus[tileset_idx]]["obj"] + tileset_file = ts_cache[locus[tileset_idx]]["path"] + elif locus[tileset_idx].endswith(".cool"): + tileset_file = path.join("data", locus[tileset_idx]) else: try: - tileset = Tileset.objects.get( - uuid=locus[tileset_idx] - ) + tileset = Tileset.objects.get(uuid=locus[tileset_idx]) tileset_file = tileset.datafile.path ts_cache[locus[tileset_idx]] = { "obj": tileset, - "path": tileset_file + "path": tileset_file, } except AttributeError: - return JsonResponse({ - 'error': 'Tileset ({}) does not exist'.format( - locus[tileset_idx] - ), - }, status=400) + return JsonResponse( + { + "error": "Tileset ({}) does not exist".format( + locus[tileset_idx] + ) + }, + status=400, + ) except Tileset.DoesNotExist: - if locus[tileset_idx].startswith('osm'): + if locus[tileset_idx].startswith("osm"): new_filetype = locus[tileset_idx] else: - return JsonResponse({ - 'error': 'Tileset ({}) does not exist'.format( - locus[tileset_idx] - ), - }, status=400) + return JsonResponse( + { + "error": "Tileset ({}) does not exist".format( + locus[tileset_idx] + ) + }, + status=400, + ) else: - return JsonResponse({ - 'error': 'Tileset not specified', - }, status=400) + return JsonResponse({"error": "Tileset not specified"}, status=400) # Get the dimensions of the snippets (i.e., width and height in px) inset_dim = ( locus[zoom_level_idx + 1] - if ( - len(locus) >= zoom_level_idx + 2 and - locus[zoom_level_idx + 1] - ) + if (len(locus) >= zoom_level_idx + 2 and locus[zoom_level_idx + 1]) else None ) out_dim = dims if inset_dim is None else inset_dim # Make sure out dim (in pixel) is not too large - if ( - (is_cool and out_dim > hss.SNIPPET_MAT_MAX_OUT_DIM) or - (not is_cool and out_dim > hss.SNIPPET_IMG_MAX_OUT_DIM) + if (is_cool and out_dim > hss.SNIPPET_MAT_MAX_OUT_DIM) or ( + not is_cool and out_dim > hss.SNIPPET_IMG_MAX_OUT_DIM ): - return JsonResponse({ - 'error': 'Snippet too large', - 'error_message': str(SnippetTooLarge()) - }, status=400) + return JsonResponse( + { + "error": "Snippet too large", + "error_message": str(SnippetTooLarge()), + }, + status=400, + ) if tileset_file not in loci_lists: loci_lists[tileset_file] = {} @@ -322,12 +321,12 @@ def get_fragments_by_loci(request): # Get max abs dim in base pairs max_abs_dim = max(locus[2] - locus[1], locus[5] - locus[4]) - with h5py.File(tileset_file, 'r') as f: + with h5py.File(tileset_file, "r") as f: # get base resolution (bin size) of cooler file - if 'resolutions' in f: + if "resolutions" in f: # v2 resolutions = sorted( - [int(key) for key in f['resolutions'].keys()] + [int(key) for key in f["resolutions"].keys()] ) closest_res = 0 for i, res in enumerate(resolutions): @@ -341,8 +340,8 @@ def get_fragments_by_loci(request): ) else: # v1 - max_zoom = f.attrs['max-zoom'] - bin_size = int(f[str(max_zoom)].attrs['bin-size']) + max_zoom = f.attrs["max-zoom"] + bin_size = int(f[str(max_zoom)].attrs["bin-size"]) # Find closest zoom level if `zoomout_level < 0` # Assuming resolutions of powers of 2 @@ -369,7 +368,7 @@ def get_fragments_by_loci(request): if zoomout_level not in loci_lists[tileset_file]: loci_lists[tileset_file][zoomout_level] = [] - locus_id = '.'.join(map(str, locus)) + locus_id = ".".join(map(str, locus)) loci_lists[tileset_file][zoomout_level].append( locus[0:tileset_idx] + [total_valid_loci, inset_dim, locus_id] @@ -380,52 +379,50 @@ def get_fragments_by_loci(request): new_filetype = ( tileset.filetype if tileset - else tileset_file[tileset_file.rfind('.') + 1:] + else tileset_file[tileset_file.rfind(".") + 1 :] ) if filetype is None: filetype = new_filetype if filetype != new_filetype: - return JsonResponse({ - 'error': ( - 'Multiple file types per query are not supported yet.' - ) - }, status=400) + return JsonResponse( + {"error": ("Multiple file types per query are not supported yet.")}, + status=400, + ) total_valid_loci += 1 except Exception as e: - return JsonResponse({ - 'error': 'Could not convert loci.', - 'error_message': str(e) - }, status=500) + return JsonResponse( + {"error": "Could not convert loci.", "error_message": str(e)}, status=500 + ) mat_idx = list(range(len(loci_ids))) # Get a unique string for caching dump = ( - json.dumps(loci, sort_keys=True) + - str(forced_rep_idx) + - str(dims) + - str(padding) + - str(no_balance) + - str(percentile) + - str(precision) + - str(ignore_diags) + - str(no_normalize) + - str(aggregate) + - str(aggregation_method) + - str(max_previews) + - str(encoding) + - str(representatives) + json.dumps(loci, sort_keys=True) + + str(forced_rep_idx) + + str(dims) + + str(padding) + + str(no_balance) + + str(percentile) + + str(precision) + + str(ignore_diags) + + str(no_normalize) + + str(aggregate) + + str(aggregation_method) + + str(max_previews) + + str(encoding) + + str(representatives) ) - uuid = hashlib.md5(dump.encode('utf-8')).hexdigest() + uuid = hashlib.md5(dump.encode("utf-8")).hexdigest() # Check if something is cached if not no_cache: try: - results = rdb.get('frag_by_loci_%s' % uuid) + results = rdb.get("frag_by_loci_%s" % uuid) if results: return JsonResponse(pickle.loads(results)) except: @@ -436,7 +433,7 @@ def get_fragments_by_loci(request): try: for dataset in loci_lists: for zoomout_level in loci_lists[dataset]: - if filetype == 'cooler' or filetype == 'cool': + if filetype == "cooler" or filetype == "cool": raw_matrices = get_frag_by_loc_from_cool( dataset, loci_lists[dataset][zoomout_level], @@ -453,12 +450,12 @@ def get_fragments_by_loci(request): for i, matrix in enumerate(raw_matrices): idx = loci_lists[dataset][zoomout_level][i][6] matrices[idx] = matrix - data_types[idx] = 'matrix' + data_types[idx] = "matrix" - if filetype == 'imtiles' or filetype == 'osm-image': + if filetype == "imtiles" or filetype == "osm-image": extractor = ( get_frag_by_loc_from_imtiles - if filetype == 'imtiles' + if filetype == "imtiles" else get_frag_by_loc_from_osm ) @@ -475,36 +472,31 @@ def get_fragments_by_loci(request): matrices[idx] = im - data_types[idx] = 'matrix' + data_types[idx] = "matrix" except Exception as ex: raise - return JsonResponse({ - 'error': 'Could not retrieve fragments.', - 'error_message': str(ex) - }, status=500) + return JsonResponse( + {"error": "Could not retrieve fragments.", "error_message": str(ex)}, + status=500, + ) if aggregate and len(matrices) > 1: try: cover, previews_1d, previews_2d = aggregate_frags( - matrices, - loci_ids, - aggregation_method, - max_previews, + matrices, loci_ids, aggregation_method, max_previews ) matrices = [cover] mat_idx = [] if previews_1d is not None: - previews = np.split( - previews_1d, range(1, previews_1d.shape[0]) - ) + previews = np.split(previews_1d, range(1, previews_1d.shape[0])) data_types = [data_types[0]] except Exception as ex: raise - return JsonResponse({ - 'error': 'Could not aggregate fragments.', - 'error_message': str(ex) - }, status=500) + return JsonResponse( + {"error": "Could not aggregate fragments.", "error_message": str(ex)}, + status=500, + ) if representatives and len(matrices) > 1: if forced_rep_idx and len(forced_rep_idx) <= len(matrices): @@ -521,12 +513,15 @@ def get_fragments_by_loci(request): data_types = [data_types[0]] * len(rep_frags) except Exception as ex: raise - return JsonResponse({ - 'error': 'Could get representative fragments.', - 'error_message': str(ex) - }, status=500) + return JsonResponse( + { + "error": "Could get representative fragments.", + "error_message": str(ex), + }, + status=500, + ) - if encoding != 'b64' and encoding != 'image': + if encoding != "b64" and encoding != "image": # Adjust precision and convert to list for i, matrix in enumerate(matrices): if precision > 0: @@ -540,25 +535,25 @@ def get_fragments_by_loci(request): previews_2d[i] = preview_2d.tolist() # Encode matrix if required - if encoding == 'b64': + if encoding == "b64": for i, matrix in enumerate(matrices): id = loci_ids[mat_idx[i]] - data_types[i] = 'dataUrl' + data_types[i] = "dataUrl" if not no_cache and id: mat_b64 = None try: - mat_b64 = rdb.get('im_b64_%s' % id) + mat_b64 = rdb.get("im_b64_%s" % id) if mat_b64 is not None: - matrices[i] = mat_b64.decode('ascii') + matrices[i] = mat_b64.decode("ascii") continue except: pass - mat_b64 = pybase64.b64encode(np_to_png(matrix)).decode('ascii') + mat_b64 = pybase64.b64encode(np_to_png(matrix)).decode("ascii") if not no_cache: try: - rdb.set('im_b64_%s' % id, mat_b64, 60 * 30) + rdb.set("im_b64_%s" % id, mat_b64, 60 * 30) except Exception as ex: # error caching a tile # log the error and carry forward, this isn't critical @@ -568,120 +563,120 @@ def get_fragments_by_loci(request): if max_previews > 0: for i, preview in enumerate(previews): - previews[i] = pybase64.b64encode( - np_to_png(preview) - ).decode('ascii') + previews[i] = pybase64.b64encode(np_to_png(preview)).decode("ascii") for i, preview_2d in enumerate(previews_2d): - previews_2d[i] = pybase64.b64encode( - np_to_png(preview_2d) - ).decode('ascii') + previews_2d[i] = pybase64.b64encode(np_to_png(preview_2d)).decode( + "ascii" + ) # Create results results = { - 'fragments': matrices, - 'indices': [int(i) for i in mat_idx], - 'dataTypes': data_types, + "fragments": matrices, + "indices": [int(i) for i in mat_idx], + "dataTypes": data_types, } # Return Y aggregates as 1D previews on demand if max_previews > 0: - results['previews'] = previews - results['previews2d'] = previews_2d + results["previews"] = previews + results["previews2d"] = previews_2d # Cache results for 30 minutes try: - rdb.set('frag_by_loci_%s' % uuid, pickle.dumps(results), 60 * 30) + rdb.set("frag_by_loci_%s" % uuid, pickle.dumps(results), 60 * 30) except Exception as ex: # error caching a tile # log the error and carry forward, this isn't critical logger.warn(ex) - if encoding == 'image': + if encoding == "image": if len(matrices) == 1: return HttpResponse( np_to_png(grey_to_rgb(matrices[0], to_rgba=True)), - content_type='image/png' + content_type="image/png", ) else: ims = [] for i, matrix in enumerate(matrices): - ims.append({ - 'name': '{}.png'.format(i), - 'bytes': np_to_png(grey_to_rgb(matrix, to_rgba=True)) - }) + ims.append( + { + "name": "{}.png".format(i), + "bytes": np_to_png(grey_to_rgb(matrix, to_rgba=True)), + } + ) return blob_to_zip(ims, to_resp=True) return JsonResponse(results) -@api_view(['GET']) +@api_view(["GET"]) @authentication_classes((CsrfExemptSessionAuthentication, BasicAuthentication)) def fragments_by_chr(request): - chrom = request.GET.get('chrom', False) - cooler_file = request.GET.get('cooler', False) - loop_list = request.GET.get('loop-list', False) + chrom = request.GET.get("chrom", False) + cooler_file = request.GET.get("cooler", False) + loop_list = request.GET.get("loop-list", False) if cooler_file: - if cooler_file.endswith('.cool'): - cooler_file = path.join('data', cooler_file) + if cooler_file.endswith(".cool"): + cooler_file = path.join("data", cooler_file) else: try: cooler_file = Tileset.objects.get(uuid=cooler_file).datafile.path except AttributeError: - return JsonResponse({ - 'error': 'Cooler file not in database', - }, status=500) + return JsonResponse( + {"error": "Cooler file not in database"}, status=500 + ) else: - return JsonResponse({ - 'error': 'Cooler file not specified', - }, status=500) + return JsonResponse({"error": "Cooler file not specified"}, status=500) try: - measures = request.GET.getlist('measures', []) + measures = request.GET.getlist("measures", []) except ValueError: measures = [] try: - zoomout_level = int(request.GET.get('zoomout-level', -1)) + zoomout_level = int(request.GET.get("zoomout-level", -1)) except ValueError: zoomout_level = -1 try: - limit = int(request.GET.get('limit', -1)) + limit = int(request.GET.get("limit", -1)) except ValueError: limit = -1 try: - precision = int(request.GET.get('precision', False)) + precision = int(request.GET.get("precision", False)) except ValueError: precision = False try: - no_cache = bool(request.GET.get('no-cache', False)) + no_cache = bool(request.GET.get("no-cache", False)) except ValueError: no_cache = False try: - for_config = bool(request.GET.get('for-config', False)) + for_config = bool(request.GET.get("for-config", False)) except ValueError: for_config = False # Get a unique string for the URL query string uuid = hashlib.md5( - '-'.join([ - cooler_file, - chrom, - loop_list, - str(limit), - str(precision), - str(zoomout_level) - ]) + "-".join( + [ + cooler_file, + chrom, + loop_list, + str(limit), + str(precision), + str(zoomout_level), + ] + ) ).hexdigest() # Check if something is cached if not no_cache: try: - results = rdb.get('frag_by_chrom_%s' % uuid) + results = rdb.get("frag_by_chrom_%s" % uuid) if results: return JsonResponse(pickle.loads(results)) @@ -691,13 +686,12 @@ def fragments_by_chr(request): # Get relative loci try: (loci_rel, chroms) = get_intra_chr_loops_from_looplist( - path.join('data', loop_list), chrom + path.join("data", loop_list), chrom ) except Exception as e: - return JsonResponse({ - 'error': 'Could not retrieve loci.', - 'error_message': str(e) - }, status=500) + return JsonResponse( + {"error": "Could not retrieve loci.", "error_message": str(e)}, status=500 + ) # Convert to chromosome-relative loci list loci_rel_chroms = np.column_stack( @@ -710,15 +704,13 @@ def fragments_by_chr(request): # Get fragments try: matrices = get_frag_by_loc( - cooler_file, - loci_rel_chroms, - zoomout_level=zoomout_level + cooler_file, loci_rel_chroms, zoomout_level=zoomout_level ) except Exception as e: - return JsonResponse({ - 'error': 'Could not retrieve fragments.', - 'error_message': str(e) - }, status=500) + return JsonResponse( + {"error": "Could not retrieve fragments.", "error_message": str(e)}, + status=500, + ) if precision > 0: matrices = np.around(matrices, decimals=precision) @@ -738,20 +730,16 @@ def fragments_by_chr(request): measures_values = [] for measure in measures: - if measure == 'distance-to-diagonal': - measures_values.append( - calc_measure_dtd(matrix, loci_struct[i]) - ) + if measure == "distance-to-diagonal": + measures_values.append(calc_measure_dtd(matrix, loci_struct[i])) - if measure == 'size': - measures_values.append( - calc_measure_size(matrix, loci_struct[i]) - ) + if measure == "size": + measures_values.append(calc_measure_size(matrix, loci_struct[i])) - if measure == 'noise': + if measure == "noise": measures_values.append(calc_measure_noise(matrix)) - if measure == 'sharpness': + if measure == "sharpness": measures_values.append(calc_measure_sharpness(matrix)) frag_obj = { @@ -759,54 +747,52 @@ def fragments_by_chr(request): } frag_obj.update(loci_struct[i]) - frag_obj.update({ - "measures": measures_values - }) + frag_obj.update({"measures": measures_values}) fragments.append(frag_obj) i += 1 # Create results results = { - 'count': matrices.shape[0], - 'dims': matrices.shape[1], - 'fragments': fragments, - 'measures': measures_applied, - 'relativeLoci': True, - 'zoomoutLevel': zoomout_level + "count": matrices.shape[0], + "dims": matrices.shape[1], + "fragments": fragments, + "measures": measures_applied, + "relativeLoci": True, + "zoomoutLevel": zoomout_level, } if for_config: - results['fragmentsHeader'] = [ - 'chrom1', - 'start1', - 'end1', - 'strand1', - 'chrom2', - 'start2', - 'end2', - 'strand2' + results["fragmentsHeader"] = [ + "chrom1", + "start1", + "end1", + "strand1", + "chrom2", + "start2", + "end2", + "strand2", ] + measures_applied fragments_arr = [] for fragment in fragments: tmp = [ - fragment['chrom1'], - fragment['start1'], - fragment['end1'], - fragment['strand1'], - fragment['chrom2'], - fragment['start2'], - fragment['end2'], - fragment['strand2'], - ] + fragment['measures'] + fragment["chrom1"], + fragment["start1"], + fragment["end1"], + fragment["strand1"], + fragment["chrom2"], + fragment["start2"], + fragment["end2"], + fragment["strand2"], + ] + fragment["measures"] fragments_arr.append(tmp) - results['fragments'] = fragments_arr + results["fragments"] = fragments_arr # Cache results for 30 mins try: - rdb.set('frag_by_chrom_%s' % uuid, pickle.dumps(results), 60 * 30) + rdb.set("frag_by_chrom_%s" % uuid, pickle.dumps(results), 60 * 30) except Exception as ex: # error caching a tile # log the error and carry forward, this isn't critical @@ -815,15 +801,15 @@ def fragments_by_chr(request): return JsonResponse(results) -@api_view(['GET']) +@api_view(["GET"]) @authentication_classes((CsrfExemptSessionAuthentication, BasicAuthentication)) def loci(request): - chrom = request.GET.get('chrom', False) - loop_list = request.GET.get('loop-list', False) + chrom = request.GET.get("chrom", False) + loop_list = request.GET.get("loop-list", False) # Get relative loci (loci_rel, chroms) = get_intra_chr_loops_from_looplist( - path.join('data', loop_list), chrom + path.join("data", loop_list), chrom ) loci_rel_chroms = np.column_stack( @@ -831,8 +817,6 @@ def loci(request): ) # Create results - results = { - 'loci': rel_loci_2_obj(loci_rel_chroms) - } + results = {"loci": rel_loci_2_obj(loci_rel_chroms)} return JsonResponse(results) diff --git a/higlass_server/settings.py b/higlass_server/settings.py index ce77925a..e76e38b3 100644 --- a/higlass_server/settings.py +++ b/higlass_server/settings.py @@ -19,8 +19,8 @@ from django.core.exceptions import ImproperlyConfigured # Build paths inside the project like this: os.path.join(BASE_DIR, ...) -if 'HIGLASS_SERVER_BASE_DIR' in os.environ: - base_dir = os.environ['HIGLASS_SERVER_BASE_DIR'] +if "HIGLASS_SERVER_BASE_DIR" in os.environ: + base_dir = os.environ["HIGLASS_SERVER_BASE_DIR"] if op.exists(base_dir): BASE_DIR = base_dir @@ -29,26 +29,24 @@ else: BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -if 'HIGLASS_CACHE_DIR' in os.environ: +if "HIGLASS_CACHE_DIR" in os.environ: # cache uploaded files # useful when using a mounted media directory - CACHE_DIR = os.environ['HIGLASS_CACHE_DIR'] + CACHE_DIR = os.environ["HIGLASS_CACHE_DIR"] else: CACHE_DIR = None -if 'MAX_BAM_TILE_WIDTH' in os.environ: - MAX_BAM_TILE_WIDTH = int(os.environ['MAX_BAM_TILE_WIDTH']) +if "MAX_BAM_TILE_WIDTH" in os.environ: + MAX_BAM_TILE_WIDTH = int(os.environ["MAX_BAM_TILE_WIDTH"]) else: MAX_BAM_TILE_WIDTH = int(1e5) -local_settings_file_path = os.path.join( - BASE_DIR, 'config.json' -) +local_settings_file_path = os.path.join(BASE_DIR, "config.json") # load config.json try: - with open(local_settings_file_path, 'r') as f: + with open(local_settings_file_path, "r") as f: local_settings = json.load(f) except IOError: local_settings = {} @@ -60,9 +58,7 @@ def get_setting(name, default=None, settings=local_settings): """Get the local settings variable or return explicit exception""" if default is None: - raise ImproperlyConfigured( - "Missing default value for '{0}'".format(name) - ) + raise ImproperlyConfigured("Missing default value for '{0}'".format(name)) # Try looking up setting in `config.json` first try: @@ -79,103 +75,96 @@ def get_setting(name, default=None, settings=local_settings): if default is not None: return default else: - raise ImproperlyConfigured( - "Missing setting for '{0}' setting".format(name) - ) + raise ImproperlyConfigured("Missing setting for '{0}' setting".format(name)) # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = get_setting('SECRET_KEY', slugid.nice()) +SECRET_KEY = get_setting("SECRET_KEY", slugid.nice()) # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = get_setting('DEBUG', False) +DEBUG = get_setting("DEBUG", False) -ALLOWED_HOSTS = [ - '*', -] +ALLOWED_HOSTS = ["*"] -if 'SITE_URL' in os.environ: - ALLOWED_HOSTS += [os.environ['SITE_URL']] +if "SITE_URL" in os.environ: + ALLOWED_HOSTS += [os.environ["SITE_URL"]] # this specifies where uploaded files will be place # (e.g. BASE_DIR/media/uplaods/file.x) -MEDIA_URL = 'media/' +MEDIA_URL = "media/" -if 'HIGLASS_MEDIA_ROOT' in os.environ: - MEDIA_ROOT = os.environ['HIGLASS_MEDIA_ROOT'] +if "HIGLASS_MEDIA_ROOT" in os.environ: + MEDIA_ROOT = os.environ["HIGLASS_MEDIA_ROOT"] else: - MEDIA_ROOT = os.path.join(BASE_DIR, 'media') + MEDIA_ROOT = os.path.join(BASE_DIR, "media") -if 'HTTPFS_HTTP_DIR' in os.environ: - HTTPFS_HTTP_DIR = os.environ['HTTPFS_HTTP_DIR'] +if "HTTPFS_HTTP_DIR" in os.environ: + HTTPFS_HTTP_DIR = os.environ["HTTPFS_HTTP_DIR"] else: - HTTPFS_HTTP_DIR = os.path.join(MEDIA_ROOT, 'http') + HTTPFS_HTTP_DIR = os.path.join(MEDIA_ROOT, "http") -if 'HTTPFS_HTTPS_DIR' in os.environ: - HTTPFS_HTTPS_DIR = os.environ['HTTPFS_HTTPS_DIR'] +if "HTTPFS_HTTPS_DIR" in os.environ: + HTTPFS_HTTPS_DIR = os.environ["HTTPFS_HTTPS_DIR"] else: - HTTPFS_HTTPS_DIR = os.path.join(MEDIA_ROOT, 'https') + HTTPFS_HTTPS_DIR = os.path.join(MEDIA_ROOT, "https") -if 'HTTPFS_FTP_DIR' in os.environ: - HTTPFS_FTP_DIR = os.environ['HTTPFS_FTP_DIR'] +if "HTTPFS_FTP_DIR" in os.environ: + HTTPFS_FTP_DIR = os.environ["HTTPFS_FTP_DIR"] else: - HTTPFS_FTP_DIR = os.path.join(MEDIA_ROOT, 'ftp') + HTTPFS_FTP_DIR = os.path.join(MEDIA_ROOT, "ftp") -THUMBNAILS_ROOT = os.path.join(MEDIA_ROOT, 'thumbnails') -AWS_BUCKET_MOUNT_POINT = os.path.join(MEDIA_ROOT, 'aws') -THUMBNAIL_RENDER_URL_BASE = '/app/' +THUMBNAILS_ROOT = os.path.join(MEDIA_ROOT, "thumbnails") +AWS_BUCKET_MOUNT_POINT = os.path.join(MEDIA_ROOT, "aws") +THUMBNAIL_RENDER_URL_BASE = "/app/" LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'verbose': { - 'format': - "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s", - 'datefmt': "%d/%b/%Y %H:%M:%S" - }, - 'simple': { - 'format': '%(levelname)s %(message)s' + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "verbose": { + "format": "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s", + "datefmt": "%d/%b/%Y %H:%M:%S", }, + "simple": {"format": "%(levelname)s %(message)s"}, }, - 'handlers': { - 'console': { - 'level': get_setting('LOG_LEVEL_CONSOLE', 'WARNING'), - 'class': 'logging.StreamHandler', - 'formatter': 'simple' + "handlers": { + "console": { + "level": get_setting("LOG_LEVEL_CONSOLE", "WARNING"), + "class": "logging.StreamHandler", + "formatter": "simple", }, - 'file': { - 'level': get_setting('LOG_LEVEL_FILE', 'WARNING'), - 'class': 'logging.FileHandler', - 'filename': os.path.join(BASE_DIR, 'log/hgs.log'), - 'formatter': 'verbose' + "file": { + "level": get_setting("LOG_LEVEL_FILE", "WARNING"), + "class": "logging.FileHandler", + "filename": os.path.join(BASE_DIR, "log/hgs.log"), + "formatter": "verbose", }, }, - 'loggers': { - 'django': { - 'handlers': ['file'], - 'propagate': True, - 'level': get_setting('LOG_LEVEL_DJANGO', 'WARNING'), + "loggers": { + "django": { + "handlers": ["file"], + "propagate": True, + "level": get_setting("LOG_LEVEL_DJANGO", "WARNING"), }, - 'fragments': { - 'handlers': ['file'], - 'level': get_setting('LOG_LEVEL_FRAGMENTS', 'WARNING'), + "fragments": { + "handlers": ["file"], + "level": get_setting("LOG_LEVEL_FRAGMENTS", "WARNING"), }, - 'tilesets': { - 'handlers': ['file'], - 'level': get_setting('LOG_LEVEL_TILESETS', 'WARNING'), + "tilesets": { + "handlers": ["file"], + "level": get_setting("LOG_LEVEL_TILESETS", "WARNING"), }, - } + }, } if DEBUG: # make all loggers use the console. - for logger in LOGGING['loggers']: - LOGGING['loggers'][logger]['handlers'] = ['console'] + for logger in LOGGING["loggers"]: + LOGGING["loggers"][logger]["handlers"] = ["console"] -if 'REDIS_HOST' in os.environ and 'REDIS_PORT' in os.environ: - REDIS_HOST = os.environ['REDIS_HOST'] - REDIS_PORT = os.environ['REDIS_PORT'] +if "REDIS_HOST" in os.environ and "REDIS_PORT" in os.environ: + REDIS_HOST = os.environ["REDIS_HOST"] + REDIS_PORT = os.environ["REDIS_PORT"] else: REDIS_HOST = None REDIS_PORT = None @@ -185,78 +174,74 @@ def get_setting(name, default=None, settings=local_settings): # Application definition INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'rest_framework', - 'tilesets.apps.TilesetsConfig', - 'fragments.app.FragmentsConfig', - 'rest_framework_swagger', - 'corsheaders', - 'guardian' + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "rest_framework", + "tilesets.apps.TilesetsConfig", + "fragments.app.FragmentsConfig", + "rest_framework_swagger", + "corsheaders", + "guardian", ] # We want to avoid loading into memory -FILE_UPLOAD_HANDLERS = [ - 'django.core.files.uploadhandler.TemporaryFileUploadHandler' -] +FILE_UPLOAD_HANDLERS = ["django.core.files.uploadhandler.TemporaryFileUploadHandler"] MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'corsheaders.middleware.CorsMiddleware', - 'django.middleware.common.CommonMiddleware', + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "corsheaders.middleware.CorsMiddleware", + "django.middleware.common.CommonMiddleware", # 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", # 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] AUTHENTICATION_BACKENDS = ( - 'django.contrib.auth.backends.ModelBackend', # this is default - 'guardian.backends.ObjectPermissionBackend', + "django.contrib.auth.backends.ModelBackend", # this is default + "guardian.backends.ObjectPermissionBackend", ) CORS_ORIGIN_ALLOW_ALL = True # CORS_ALLOW_CREDENTIALS = False -CORS_ORIGIN_WHITELIST = [ - 'http://134.174.140.208:9000' -] +CORS_ORIGIN_WHITELIST = ["http://134.174.140.208:9000"] # CORS_ALLOW_HEADERS = default_headers -ROOT_URLCONF = 'higlass_server.urls' +ROOT_URLCONF = "higlass_server.urls" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ] }, - }, + } ] -WSGI_APPLICATION = 'higlass_server.wsgi.application' +WSGI_APPLICATION = "higlass_server.wsgi.application" # Database # https://docs.djangoproject.com/en/1.10/ref/settings/#databases DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": os.path.join(BASE_DIR, "db.sqlite3"), } } @@ -264,33 +249,26 @@ def get_setting(name, default=None, settings=local_settings): # Password validation # https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators -AUTH_PASSWORD_VALIDATORS = [{ - 'NAME': - 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', -}, { - 'NAME': - 'django.contrib.auth.password_validation.MinimumLengthValidator', -}, { - 'NAME': - 'django.contrib.auth.password_validation.CommonPasswordValidator', -}, { - 'NAME': - 'django.contrib.auth.password_validation.NumericPasswordValidator', -}] +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator" + }, + {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"}, + {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"}, + {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"}, +] REST_FRAMEWORK = { - 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', - 'PAGE_SIZE': 10, - 'DEFAULT_RENDERER_CLASSES': ( - 'rest_framework.renderers.JSONRenderer', - ) + "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination", + "PAGE_SIZE": 10, + "DEFAULT_RENDERER_CLASSES": ("rest_framework.renderers.JSONRenderer",), } # Internationalization # https://docs.djangoproject.com/en/1.10/topics/i18n/ -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" -TIME_ZONE = 'UTC' +TIME_ZONE = "UTC" USE_I18N = True @@ -298,33 +276,33 @@ def get_setting(name, default=None, settings=local_settings): USE_TZ = True -UPLOAD_ENABLED = get_setting('UPLOAD_ENABLED', True) -PUBLIC_UPLOAD_ENABLED = get_setting('PUBLIC_UPLOAD_ENABLED', True) +UPLOAD_ENABLED = get_setting("UPLOAD_ENABLED", True) +PUBLIC_UPLOAD_ENABLED = get_setting("PUBLIC_UPLOAD_ENABLED", True) -SNIPPET_MAT_MAX_OUT_DIM = get_setting('SNIPPET_MAT_MAX_OUT_DIM', 512) -SNIPPET_MAT_MAX_DATA_DIM = get_setting('SNIPPET_MAT_MAX_DATA_DIM', 4096) -SNIPPET_IMG_MAX_OUT_DIM = get_setting('SNIPPET_IMG_MAX_OUT_DIM', 1024) -SNIPPET_OSM_MAX_DATA_DIM = get_setting('SNIPPET_OSM_MAX_DATA_DIM', 2048) -SNIPPET_IMT_MAX_DATA_DIM = get_setting('SNIPPET_IMT_MAX_DATA_DIM', 2048) +SNIPPET_MAT_MAX_OUT_DIM = get_setting("SNIPPET_MAT_MAX_OUT_DIM", 512) +SNIPPET_MAT_MAX_DATA_DIM = get_setting("SNIPPET_MAT_MAX_DATA_DIM", 4096) +SNIPPET_IMG_MAX_OUT_DIM = get_setting("SNIPPET_IMG_MAX_OUT_DIM", 1024) +SNIPPET_OSM_MAX_DATA_DIM = get_setting("SNIPPET_OSM_MAX_DATA_DIM", 2048) +SNIPPET_IMT_MAX_DATA_DIM = get_setting("SNIPPET_IMT_MAX_DATA_DIM", 2048) # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.10/howto/static-files/ -STATIC_URL = '/hgs-static/' -STATIC_ROOT = 'hgs-static/' +STATIC_URL = "/hgs-static/" +STATIC_ROOT = "hgs-static/" -if 'APP_BASEPATH' in os.environ: +if "APP_BASEPATH" in os.environ: # https://stackoverflow.com/questions/44987110/django-in-subdirectory-admin-site-is-not-working USE_X_FORWARDED_HOST = True - FORCE_SCRIPT_NAME = os.environ['APP_BASEPATH'] - SESSION_COOKIE_PATH = os.environ['APP_BASEPATH'] - LOGIN_REDIRECT_URL = os.environ['APP_BASEPATH'] - LOGOUT_REDIRECT_URL = os.environ['APP_BASEPATH'] + FORCE_SCRIPT_NAME = os.environ["APP_BASEPATH"] + SESSION_COOKIE_PATH = os.environ["APP_BASEPATH"] + LOGIN_REDIRECT_URL = os.environ["APP_BASEPATH"] + LOGOUT_REDIRECT_URL = os.environ["APP_BASEPATH"] - STATIC_URL = op.join(os.environ['APP_BASEPATH'], 'hgs-static') + "/" + STATIC_URL = op.join(os.environ["APP_BASEPATH"], "hgs-static") + "/" -ADMIN_URL = r'^admin/' +ADMIN_URL = r"^admin/" # STATICFILES_DIRS = ( # os.path.join(BASE_DIR, 'static'), @@ -332,4 +310,4 @@ def get_setting(name, default=None, settings=local_settings): # TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' -NOSE_ARGS = ['--nocapture', '--nologcapture'] +NOSE_ARGS = ["--nocapture", "--nologcapture"] diff --git a/higlass_server/settings_test.py b/higlass_server/settings_test.py index e3645fe6..b4010fae 100644 --- a/higlass_server/settings_test.py +++ b/higlass_server/settings_test.py @@ -1,9 +1,9 @@ from .settings import * DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db_test.sqlite3'), + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": os.path.join(BASE_DIR, "db_test.sqlite3"), } } diff --git a/higlass_server/tests.py b/higlass_server/tests.py index f60bcc0c..a5f4d551 100644 --- a/higlass_server/tests.py +++ b/higlass_server/tests.py @@ -4,93 +4,135 @@ import tilesets.models as tm + class CommandlineTest(unittest.TestCase): def setUp(self): # TODO: There is probably a better way to clear data from previous test runs. Is it even necessary? # self.assertRun('python manage.py flush --noinput --settings=higlass_server.settings_test') pass - def assertRun(self, command, output_res=[]): - output = subprocess.check_output(command , shell=True).decode('utf-8').strip() + def assertRun(self, command, output_res=[]): + output = subprocess.check_output(command, shell=True).decode("utf-8").strip() for output_re in output_res: self.assertRegexpMatches(output, output_re) return output def test_hello(self): - self.assertRun('echo "hello?"', [r'hello']) + self.assertRun('echo "hello?"', [r"hello"]) def test_bamfile_upload_with_index(self): - settings = 'higlass_server.settings_test' + settings = "higlass_server.settings_test" uid = slugid.nice() - self.assertRun('python manage.py ingest_tileset' + - ' --filename data/SRR1770413.mismatched_bai.bam' + - ' --indexfile data/SRR1770413.different_index_filename.bai' + - ' --datatype reads' + - ' --filetype bam' + - ' --uid '+uid+' --settings='+settings) + self.assertRun( + "python manage.py ingest_tileset" + + " --filename data/SRR1770413.mismatched_bai.bam" + + " --indexfile data/SRR1770413.different_index_filename.bai" + + " --datatype reads" + + " --filetype bam" + + " --uid " + + uid + + " --settings=" + + settings + ) - self.assertRun('python manage.py shell ' + - '--settings ' + settings + - ' --command="' + - 'import tilesets.models as tm; '+ - f'o = tm.Tileset.objects.get(uuid=\'{uid}\');' - 'print(o.indexfile)"', '.bai$') + self.assertRun( + "python manage.py shell " + + "--settings " + + settings + + ' --command="' + + "import tilesets.models as tm; " + + f"o = tm.Tileset.objects.get(uuid='{uid}');" + 'print(o.indexfile)"', + ".bai$", + ) def test_bamfile_upload_without_index(self): - settings = 'higlass_server.settings_test' + settings = "higlass_server.settings_test" uid = slugid.nice() - self.assertRun('python manage.py ingest_tileset' + - ' --filename data/SRR1770413.sorted.short.bam' + - ' --datatype reads' + - ' --filetype bam' + - ' --uid '+uid+' --settings='+settings) + self.assertRun( + "python manage.py ingest_tileset" + + " --filename data/SRR1770413.sorted.short.bam" + + " --datatype reads" + + " --filetype bam" + + " --uid " + + uid + + " --settings=" + + settings + ) - self.assertRun('python manage.py shell ' + - '--settings ' + settings + - ' --command="' + - 'import tilesets.models as tm; '+ - f'o = tm.Tileset.objects.get(uuid=\'{uid}\');' - 'print(o.indexfile)"', '.bai$') + self.assertRun( + "python manage.py shell " + + "--settings " + + settings + + ' --command="' + + "import tilesets.models as tm; " + + f"o = tm.Tileset.objects.get(uuid='{uid}');" + 'print(o.indexfile)"', + ".bai$", + ) def test_cli_upload(self): - cooler = 'dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool' - settings = 'higlass_server.settings_test' - id = 'cli-test' - self.assertRun('python manage.py ingest_tileset --filename data/'+cooler+' --datatype matrix --filetype cooler --uid '+id+' --settings='+settings) - self.assertRun('curl -s http://localhost:6000/api/v1/tileset_info/?d='+id, - [r'"name": "'+cooler+'"']) - self.assertRun('curl -s http://localhost:6000/api/v1/tiles/?d='+id+'.1.1.1', - [r'"'+id+'.1.1.1":', - r'"max_value": 2.0264008045196533', - r'"min_value": 0.0', - r'"dense": "JTInPwAA']) + cooler = "dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool" + settings = "higlass_server.settings_test" + id = "cli-test" + self.assertRun( + "python manage.py ingest_tileset --filename data/" + + cooler + + " --datatype matrix --filetype cooler --uid " + + id + + " --settings=" + + settings + ) + self.assertRun( + "curl -s http://localhost:6000/api/v1/tileset_info/?d=" + id, + [r'"name": "' + cooler + '"'], + ) + self.assertRun( + "curl -s http://localhost:6000/api/v1/tiles/?d=" + id + ".1.1.1", + [ + r'"' + id + '.1.1.1":', + r'"max_value": 2.0264008045196533', + r'"min_value": 0.0', + r'"dense": "JTInPwAA', + ], + ) def test_cli_huge_upload(self): - cooler = 'huge.fake.cool' - with open('data/'+cooler, 'w') as file: + cooler = "huge.fake.cool" + with open("data/" + cooler, "w") as file: file.truncate(1024 ** 3) - settings = 'higlass_server.settings_test' - id = 'cli-huge-test' - self.assertRun('python manage.py ingest_tileset --filename data/'+cooler+' --datatype foo --filetype bar --uid '+id+' --settings='+settings) - self.assertRun('curl -s http://localhost:6000/api/v1/tileset_info/?d='+id, - [r'"name": "'+cooler+'"']) - self.assertRun('curl -s http://localhost:6000/api/v1/tiles/?d='+id+'.1.1.1', - [r'"'+id+'.1.1.1"']) + settings = "higlass_server.settings_test" + id = "cli-huge-test" + self.assertRun( + "python manage.py ingest_tileset --filename data/" + + cooler + + " --datatype foo --filetype bar --uid " + + id + + " --settings=" + + settings + ) + self.assertRun( + "curl -s http://localhost:6000/api/v1/tileset_info/?d=" + id, + [r'"name": "' + cooler + '"'], + ) + self.assertRun( + "curl -s http://localhost:6000/api/v1/tiles/?d=" + id + ".1.1.1", + [r'"' + id + '.1.1.1"'], + ) - ''' + """ id = 'cli-coord-system-test' self.assertRun('python manage.py ingest_tileset --filename data/'+cooler+' --datatype foo --filetype bar --uid '+id+' --settings='+settings, 'coordSystem') self.assertRun('curl -s http://localhost:6000/api/v1/tileset_info/?d='+id, [r'"coordSystem": "'+cooler+'"']) - ''' + """ # TODO: check the coordSystem parameters for ingest_tileset.py def test_get_from_foreign_host_file(self): # manage.py should have been started with # export SITE_URL=somesite.com - #self.assertRun('curl -s -H "Host: someothersite.com" http://localhost:6000/api/v1/tilesets/', [r'400']) - #self.assertRun('curl -s -H "Host: somesite.com" http://localhost:6000/api/v1/tilesets/', [r'count']) + # self.assertRun('curl -s -H "Host: someothersite.com" http://localhost:6000/api/v1/tilesets/', [r'400']) + # self.assertRun('curl -s -H "Host: somesite.com" http://localhost:6000/api/v1/tilesets/', [r'count']) pass - diff --git a/higlass_server/urls.py b/higlass_server/urls.py index 44df2162..535598d9 100644 --- a/higlass_server/urls.py +++ b/higlass_server/urls.py @@ -19,7 +19,7 @@ urlpatterns = [ url(settings.ADMIN_URL, admin.site.urls), - url(r'^api/v1/', include('tilesets.urls')), - url(r'^api/v1/', include('fragments.urls')), - url(r'^', include('website.urls')), + url(r"^api/v1/", include("tilesets.urls")), + url(r"^api/v1/", include("fragments.urls")), + url(r"^", include("website.urls")), ] diff --git a/higlass_server/utils.py b/higlass_server/utils.py index 53f1e169..74a9f01f 100644 --- a/higlass_server/utils.py +++ b/higlass_server/utils.py @@ -21,9 +21,7 @@ def set(self, name, value, ex=None, px=None, nx=False, xx=False): def getRdb(): if hss.REDIS_HOST is not None: try: - rdb = redis.Redis( - host=hss.REDIS_HOST, - port=hss.REDIS_PORT) + rdb = redis.Redis(host=hss.REDIS_HOST, port=hss.REDIS_PORT) # Test server connection rdb.ping() diff --git a/manage.py b/manage.py index fb037bb3..94067cd2 100755 --- a/manage.py +++ b/manage.py @@ -3,7 +3,7 @@ import sys if __name__ == "__main__": - os.environ['DJANGO_SETTINGS_MODULE'] = "higlass_server.settings" + os.environ["DJANGO_SETTINGS_MODULE"] = "higlass_server.settings" try: from django.core.management import execute_from_command_line except ImportError: diff --git a/scripts/add_attr_to_hdf5.py b/scripts/add_attr_to_hdf5.py index 71d92aa7..8e42b3b7 100644 --- a/scripts/add_attr_to_hdf5.py +++ b/scripts/add_attr_to_hdf5.py @@ -4,29 +4,30 @@ import sys import argparse + def main(): - parser = argparse.ArgumentParser(description=""" + parser = argparse.ArgumentParser( + description=""" python add_attr_to_hdf5.py file.hdf5 attr_name attr_value Add an attribute to an HDF5 file. -""") +""" + ) - parser.add_argument('filepath') - parser.add_argument('attr_name') - parser.add_argument('attr_value') - #parser.add_argument('-o', '--options', default='yo', - # help="Some option", type='str') - #parser.add_argument('-u', '--useless', action='store_true', - # help='Another useless option') + parser.add_argument("filepath") + parser.add_argument("attr_name") + parser.add_argument("attr_value") + # parser.add_argument('-o', '--options', default='yo', + # help="Some option", type='str') + # parser.add_argument('-u', '--useless', action='store_true', + # help='Another useless option') args = parser.parse_args() with h5py.File(args.filepath) as f: f.attrs[args.attr_name] = args.attr_value - - -if __name__ == '__main__': - main() +if __name__ == "__main__": + main() diff --git a/scripts/benchmark_server.py b/scripts/benchmark_server.py index 8fa27c08..dfb38fd1 100644 --- a/scripts/benchmark_server.py +++ b/scripts/benchmark_server.py @@ -8,8 +8,10 @@ import argparse from multiprocessing import Pool + def main(): - parser = argparse.ArgumentParser(description=""" + parser = argparse.ArgumentParser( + description=""" Usage: @@ -18,48 +20,53 @@ def main(): mv db.sqlite3 api python benchmark_server.py url path tileset-id [tile_ids] -""") - - parser.add_argument('url') - parser.add_argument('tileset_id') - parser.add_argument('tile_ids', nargs='*') - parser.add_argument('--tile-id-file') - parser.add_argument('--iterations') - parser.add_argument('--at-once', action='store_true') - parser.add_argument('--multi', action='store_true') - - #parser.add_argument('-o', '--options', default='yo', - # help="Some option", type='str') - #parser.add_argument('-u', '--useless', action='store_true', - # help='Another useless option') +""" + ) + + parser.add_argument("url") + parser.add_argument("tileset_id") + parser.add_argument("tile_ids", nargs="*") + parser.add_argument("--tile-id-file") + parser.add_argument("--iterations") + parser.add_argument("--at-once", action="store_true") + parser.add_argument("--multi", action="store_true") + + # parser.add_argument('-o', '--options', default='yo', + # help="Some option", type='str') + # parser.add_argument('-u', '--useless', action='store_true', + # help='Another useless option') args = parser.parse_args() tile_ids = args.tile_ids # parse requests on the command line for tile_id in args.tile_ids: - get_url = op.join(args.url, 'tilesets/x/render/?d=' + args.tileset_id + '.' + tile_id) + get_url = op.join( + args.url, "tilesets/x/render/?d=" + args.tileset_id + "." + tile_id + ) r = requests.get(get_url) print("r:", r) - + # parse requests from a file if args.tile_id_file is not None: - with open(args.tile_id_file, 'r') as f: + with open(args.tile_id_file, "r") as f: for line in f: tile_ids += [line.strip()] if args.at_once: - url_arg = "&d=".join([args.tileset_id + '.' + tile_id for tile_id in tile_ids]) - get_url = op.join(args.url, 'tilesets/x/render/?d=' + url_arg) + url_arg = "&d=".join([args.tileset_id + "." + tile_id for tile_id in tile_ids]) + get_url = op.join(args.url, "tilesets/x/render/?d=" + url_arg) print("get_url:", get_url) r = requests.get(get_url) print("r:", r, len(r.text)) - + else: arr = [] for tile_id in tile_ids: - get_url = op.join(args.url, 'tilesets/x/render/?d=' + args.tileset_id + '.' + tile_id) + get_url = op.join( + args.url, "tilesets/x/render/?d=" + args.tileset_id + "." + tile_id + ) arr.append(get_url) if args.multi: @@ -70,7 +77,6 @@ def main(): for a in arr: requests.get(a) -if __name__ == '__main__': - main() - +if __name__ == "__main__": + main() diff --git a/scripts/format_upload_command.py b/scripts/format_upload_command.py index f4b885a9..11b3d7a9 100644 --- a/scripts/format_upload_command.py +++ b/scripts/format_upload_command.py @@ -5,8 +5,10 @@ import sys import argparse + def main(): - parser = argparse.ArgumentParser(description=""" + parser = argparse.ArgumentParser( + description=""" python format_upload_command.py formatted_filename @@ -19,25 +21,26 @@ def main(): Example output: ... -""") +""" + ) - parser.add_argument('filename') - #parser.add_argument('argument', nargs=1) - #parser.add_argument('-o', '--options', default='yo', - # help="Some option", type='str') - #parser.add_argument('-u', '--useless', action='store_true', - # help='Another useless option') + parser.add_argument("filename") + # parser.add_argument('argument', nargs=1) + # parser.add_argument('-o', '--options', default='yo', + # help="Some option", type='str') + # parser.add_argument('-u', '--useless', action='store_true', + # help='Another useless option') args = parser.parse_args() - parts = args.filename.split('-') + parts = args.filename.split("-") try: name = parts[0][:-4] year = parts[0][-4:] celltype = parts[1] enzyme = parts[2] - resolution = parts[4].split('.')[1] + resolution = parts[4].split(".")[1] out_txt = """ curl -u `cat ~/.higlass-server-login` \ @@ -46,14 +49,19 @@ def main(): -F 'datatype=matrix' \ -F 'name={name} et al. ({year}) {celltype} {enzyme} (allreps) {resolution}' \ -F 'coordSystem=hg19' \ - localhost:8000/api/v1/tilesets/""".format(filename=args.filename, name=name, year=year, celltype=celltype, enzyme=enzyme, resolution=resolution) + localhost:8000/api/v1/tilesets/""".format( + filename=args.filename, + name=name, + year=year, + celltype=celltype, + enzyme=enzyme, + resolution=resolution, + ) print(out_txt, end="") except: print("ERROR:", args.filename) - - -if __name__ == '__main__': - main() +if __name__ == "__main__": + main() diff --git a/scripts/test_aws_bigWig_fetch.py b/scripts/test_aws_bigWig_fetch.py index bada41da..edc25022 100644 --- a/scripts/test_aws_bigWig_fetch.py +++ b/scripts/test_aws_bigWig_fetch.py @@ -4,35 +4,38 @@ import tilesets.bigwig_tiles as bwt import time + def main(): - parser = argparse.ArgumentParser(description=""" + parser = argparse.ArgumentParser( + description=""" python test_bigwig_tile_fetch.py filename zoom_level tile_pos -""") +""" + ) - parser.add_argument('filename') - parser.add_argument('zoom_level') - parser.add_argument('tile_pos') - parser.add_argument('--num-requests', default=1, type=int) - #parser.add_argument('argument', nargs=1) - #parser.add_argument('-o', '--options', default='yo', - # help="Some option", type='str') - #parser.add_argument('-u', '--useless', action='store_true', - # help='Another useless option') + parser.add_argument("filename") + parser.add_argument("zoom_level") + parser.add_argument("tile_pos") + parser.add_argument("--num-requests", default=1, type=int) + # parser.add_argument('argument', nargs=1) + # parser.add_argument('-o', '--options', default='yo', + # help="Some option", type='str') + # parser.add_argument('-u', '--useless', action='store_true', + # help='Another useless option') args = parser.parse_args() print("fetching:", args.filename) if args.num_requests == 1: - tile = bwt.get_bigwig_tile_by_id(args.filename, int(args.zoom_level), - int(args.tile_pos)) + tile = bwt.get_bigwig_tile_by_id( + args.filename, int(args.zoom_level), int(args.tile_pos) + ) else: zoom_level = math.ceil(math.log(args.num_requests) / math.log(2)) for tn in range(0, args.num_requests): print("fetching:", zoom_level, tn) t1 = time.time() - tile = bwt.get_bigwig_tile_by_id(args.filename, int(zoom_level), - int(tn)) + tile = bwt.get_bigwig_tile_by_id(args.filename, int(zoom_level), int(tn)) t2 = time.time() print("fetched: {:.2f}".format(t2 - t1), "tile", len(tile)) diff --git a/tilesets/admin.py b/tilesets/admin.py index 186f6767..546d9741 100644 --- a/tilesets/admin.py +++ b/tilesets/admin.py @@ -2,38 +2,31 @@ from tilesets.models import Tileset from tilesets.models import ViewConf from tilesets.models import Project + # Register your models here. class TilesetAdmin(admin.ModelAdmin): list_display = [ - 'created', - 'uuid', - 'datafile', - 'filetype', - 'datatype', - 'coordSystem', - 'coordSystem2', - 'owner', - 'private', - 'name', + "created", + "uuid", + "datafile", + "filetype", + "datatype", + "coordSystem", + "coordSystem2", + "owner", + "private", + "name", ] class ViewConfAdmin(admin.ModelAdmin): - list_display = [ - 'created', - 'uuid', - 'higlassVersion', - ] - + list_display = ["created", "uuid", "higlassVersion"] + + class ProjectConfAdmin(admin.ModelAdmin): - list_display = [ - 'created', - 'uuid', - 'name', - 'description', - ] + list_display = ["created", "uuid", "name", "description"] admin.site.register(Tileset, TilesetAdmin) diff --git a/tilesets/apps.py b/tilesets/apps.py index 6cd9bf5b..b6055dd5 100644 --- a/tilesets/apps.py +++ b/tilesets/apps.py @@ -4,4 +4,4 @@ class TilesetsConfig(AppConfig): - name = 'tilesets' + name = "tilesets" diff --git a/tilesets/bigwig_tiles.py b/tilesets/bigwig_tiles.py index bb1db452..3666e416 100644 --- a/tilesets/bigwig_tiles.py +++ b/tilesets/bigwig_tiles.py @@ -5,6 +5,7 @@ TILE_SIZE = 1024 + def get_quadtree_depth(chromsizes): tile_size_bp = TILE_SIZE min_tile_cover = np.ceil(sum(chromsizes) / tile_size_bp) @@ -12,7 +13,7 @@ def get_quadtree_depth(chromsizes): def get_zoom_resolutions(chromsizes): - return [2**x for x in range(get_quadtree_depth(chromsizes) + 1)][::-1] + return [2 ** x for x in range(get_quadtree_depth(chromsizes) + 1)][::-1] def get_chromsizes(bwpath): @@ -29,9 +30,9 @@ def get_chromsizes(bwpath): def abs2genomic(chromsizes, start_pos, end_pos): abs_chrom_offsets = np.r_[0, np.cumsum(chromsizes.values)] - cid_lo, cid_hi = np.searchsorted(abs_chrom_offsets, - [start_pos, end_pos], - side='right') - 1 + cid_lo, cid_hi = ( + np.searchsorted(abs_chrom_offsets, [start_pos, end_pos], side="right") - 1 + ) rel_pos_lo = start_pos - abs_chrom_offsets[cid_lo] rel_pos_hi = end_pos - abs_chrom_offsets[cid_hi] start = rel_pos_lo @@ -45,7 +46,7 @@ def get_bigwig_tile(bwpath, zoom_level, start_pos, end_pos): chromsizes = get_chromsizes(bwpath) resolutions = get_zoom_resolutions(chromsizes) binsize = resolutions[zoom_level] - + arrays = [] for cid, start, end in abs2genomic(chromsizes, start_pos, end_pos): n_bins = int(np.ceil((end - start) / binsize)) @@ -53,8 +54,7 @@ def get_bigwig_tile(bwpath, zoom_level, start_pos, end_pos): chrom = chromsizes.index[cid] clen = chromsizes.values[cid] - x = bbi.fetch(bwpath, chrom, start, end, - bins=n_bins, missing=np.nan) + x = bbi.fetch(bwpath, chrom, start, end, bins=n_bins, missing=np.nan) # drop the very last bin if it is smaller than the binsize if end == clen and clen % binsize != 0: @@ -71,7 +71,7 @@ def get_bigwig_tile(bwpath, zoom_level, start_pos, end_pos): def get_bigwig_tile_by_id(bwpath, zoom_level, tile_pos): - ''' + """ Get the data for a bigWig tile given a tile id. Parameters @@ -82,7 +82,7 @@ def get_bigwig_tile_by_id(bwpath, zoom_level, tile_pos): The zoom level to get the data for tile_pos: int The position of the tile - ''' + """ max_depth = get_quadtree_depth(get_chromsizes(bwpath)) tile_size = TILE_SIZE * 2 ** (max_depth - zoom_level) @@ -90,4 +90,3 @@ def get_bigwig_tile_by_id(bwpath, zoom_level, tile_pos): end_pos = start_pos + tile_size return get_bigwig_tile(bwpath, zoom_level, start_pos, end_pos) - diff --git a/tilesets/chromsizes.py b/tilesets/chromsizes.py index 7f6aa84a..5b641850 100644 --- a/tilesets/chromsizes.py +++ b/tilesets/chromsizes.py @@ -8,18 +8,20 @@ logger = logging.getLogger(__name__) + def chromsizes_array_to_series(chromsizes): - ''' + """ Convert an array of [[chrname, size]...] values to a series indexed by chrname with size values - ''' + """ chrnames = [c[0] for c in chromsizes] chrvalues = [c[1] for c in chromsizes] return pd.Series(np.array([int(c) for c in chrvalues]), index=chrnames) + def get_multivec_chromsizes(filename): - ''' + """ Get a list of chromosome sizes from this [presumably] multivec file. @@ -32,19 +34,20 @@ def get_multivec_chromsizes(filename): ------- chromsizes: [(name:string, size:int), ...] An ordered list of chromosome names and sizes - ''' - with h5py.File(filename, 'r') as f: + """ + with h5py.File(filename, "r") as f: try: - chrom_names = [t.decode('utf-8') for t in f['chroms']['name'][:]] - chrom_lengths = f['chroms']['length'][:] + chrom_names = [t.decode("utf-8") for t in f["chroms"]["name"][:]] + chrom_lengths = f["chroms"]["length"][:] return zip(chrom_names, chrom_lengths) except Exception as e: logger.exception(e) - raise Exception( 'Error retrieving multivec chromsizes') + raise Exception("Error retrieving multivec chromsizes") + def get_cooler_chromsizes(filename): - ''' + """ Get a list of chromosome sizes from this [presumably] cooler file. @@ -57,14 +60,14 @@ def get_cooler_chromsizes(filename): ------- chromsizes: [(name:string, size:int), ...] An ordered list of chromosome names and sizes - ''' - with h5py.File(filename, 'r') as f: + """ + with h5py.File(filename, "r") as f: try: c = get_cooler(f) except Exception as e: logger.error(e) - raise Exception('Yikes... Couldn~\'t init cooler files 😵') + raise Exception("Yikes... Couldn~'t init cooler files 😵") try: data = [] @@ -73,10 +76,11 @@ def get_cooler_chromsizes(filename): return data except Exception as e: logger.error(e) - raise Exception( 'Cooler file has no `chromsizes` attribute 🤔') + raise Exception("Cooler file has no `chromsizes` attribute 🤔") + def get_tsv_chromsizes(filename): - ''' + """ Get a list of chromosome sizes from this [presumably] tsv chromsizes file file. @@ -89,10 +93,10 @@ def get_tsv_chromsizes(filename): ------- chromsizes: [(name:string, size:int), ...] An ordered list of chromosome names and sizes - ''' + """ try: - with open(filename, 'r') as f: - reader = csv.reader(f, delimiter='\t') + with open(filename, "r") as f: + reader = csv.reader(f, delimiter="\t") data = [] for row in reader: @@ -101,9 +105,6 @@ def get_tsv_chromsizes(filename): except Exception as ex: logger.error(ex) - err_msg = 'WHAT?! Could not load file %s. 😤 (%s)' % ( - filename, ex - ) + err_msg = "WHAT?! Could not load file %s. 😤 (%s)" % (filename, ex) raise Exception(err_msg) - diff --git a/tilesets/exceptions.py b/tilesets/exceptions.py index 0eb8e365..210a14c9 100644 --- a/tilesets/exceptions.py +++ b/tilesets/exceptions.py @@ -7,5 +7,5 @@ class CoolerFileBroken(APIException): status_code = 500 - default_detail = 'The cooler file is broken.' - default_code = 'cooler_file_broken' + default_detail = "The cooler file is broken." + default_code = "cooler_file_broken" diff --git a/tilesets/generate_tiles.py b/tilesets/generate_tiles.py index a14a645c..e9871b41 100644 --- a/tilesets/generate_tiles.py +++ b/tilesets/generate_tiles.py @@ -1,5 +1,6 @@ import base64 -#import tilesets.bigwig_tiles as bwt + +# import tilesets.bigwig_tiles as bwt import clodius.db_tiles as cdt import clodius.hdf_tiles as hdft import collections as col @@ -20,7 +21,7 @@ import time import tempfile import tilesets.models as tm -import tilesets.chromsizes as tcs +import tilesets.chromsizes as tcs import higlass.tilesets as hgti @@ -28,26 +29,28 @@ import higlass_server.settings as hss + def get_tileset_datatype(tileset): - ''' + """ Extract the filetype for the tileset This should be encoded in one of the tags. If there are multiple "datatype" tags, use the most recent one. - ''' + """ if tileset.datatype is not None and len(tileset.datatype) > 0: return tileset.datatype for tag in tileset.tags.all(): - parts = tag.name.split(':') - if parts[0] == 'datatype': + parts = tag.name.split(":") + if parts[0] == "datatype": return parts[1] # fall back to the filetype attribute of the tileset return tileset.datatype + def get_cached_datapath(path): - ''' + """ Check if we need to cache this file or if we have a cached copy Parameters @@ -60,7 +63,7 @@ def get_cached_datapath(path): filename: str Either the cached filename if we're caching or the original filename - ''' + """ if hss.CACHE_DIR is None: # no caching requested return path @@ -73,7 +76,7 @@ def get_cached_datapath(path): return cached_path with tempfile.TemporaryDirectory() as dirpath: - tmp = op.join(dirpath, 'cached_file') + tmp = op.join(dirpath, "cached_file") shutil.copyfile(orig_path, tmp) # check to make sure the destination directory exists @@ -86,8 +89,9 @@ def get_cached_datapath(path): return cached_path + def extract_tileset_uid(tile_id): - ''' + """ Get the tileset uid from a tile id. Should usually be all the text before the first dot. @@ -99,8 +103,8 @@ def extract_tileset_uid(tile_id): ------- tileset_uid : str The uid of the tileset that this tile comes from - ''' - tile_id_parts = tile_id.split('.') + """ + tile_id_parts = tile_id.split(".") tileset_uuid = tile_id_parts[0] return tileset_uuid @@ -109,8 +113,9 @@ def extract_tileset_uid(tile_id): def get_tileset_filetype(tileset): return tileset.filetype + def generate_1d_tiles(filename, tile_ids, get_data_function): - ''' + """ Generate a set of tiles for the given tile_ids. Parameters @@ -127,50 +132,57 @@ def generate_1d_tiles(filename, tile_ids, get_data_function): ------- tile_list: [(tile_id, tile_data),...] A list of tile_id, tile_data tuples - ''' + """ generated_tiles = [] for tile_id in tile_ids: - tile_id_parts = tile_id.split('.') + tile_id_parts = tile_id.split(".") tile_position = list(map(int, tile_id_parts[1:3])) dense = get_data_function(filename, tile_position) if len(dense): - max_dense = max(dense.reshape(-1,)) - min_dense = min(dense.reshape(-1,)) + max_dense = max(dense.reshape(-1)) + min_dense = min(dense.reshape(-1)) else: max_dense = 0 min_dense = 0 - min_f16 = np.finfo('float16').min - max_f16 = np.finfo('float16').max + min_f16 = np.finfo("float16").min + max_f16 = np.finfo("float16").max has_nan = len([d for d in dense.reshape((-1,)) if np.isnan(d)]) > 0 if ( - not has_nan and - max_dense > min_f16 and max_dense < max_f16 and - min_dense > min_f16 and min_dense < max_f16 + not has_nan + and max_dense > min_f16 + and max_dense < max_f16 + and min_dense > min_f16 + and min_dense < max_f16 ): tile_value = { - 'dense': base64.b64encode(dense.reshape((-1,)).astype('float16')).decode('utf-8'), - 'dtype': 'float16', - 'shape': dense.shape + "dense": base64.b64encode( + dense.reshape((-1,)).astype("float16") + ).decode("utf-8"), + "dtype": "float16", + "shape": dense.shape, } else: tile_value = { - 'dense': base64.b64encode(dense.reshape((-1,)).astype('float32')).decode('utf-8'), - 'dtype': 'float32', - 'shape': dense.shape + "dense": base64.b64encode( + dense.reshape((-1,)).astype("float32") + ).decode("utf-8"), + "dtype": "float32", + "shape": dense.shape, } generated_tiles += [(tile_id, tile_value)] return generated_tiles + def get_chromsizes(tileset): - ''' + """ Get a set of chromsizes matching the coordSystem of this tileset. @@ -184,20 +196,22 @@ def get_chromsizes(tileset): A set of chromsizes to be used with this bigWig file. None if no chromsizes tileset with this coordSystem exists or if two exist with this coordSystem. - ''' + """ if tileset.coordSystem is None or len(tileset.coordSystem) == None: return None try: - chrom_info_tileset = tm.Tileset.objects.get(coordSystem=tileset.coordSystem, - datatype='chromsizes') + chrom_info_tileset = tm.Tileset.objects.get( + coordSystem=tileset.coordSystem, datatype="chromsizes" + ) except: return None return tcs.get_tsv_chromsizes(chrom_info_tileset.datafile.path) + def generate_hitile_tiles(tileset, tile_ids): - ''' + """ Generate tiles from a hitile file. Parameters @@ -212,19 +226,15 @@ def generate_hitile_tiles(tileset, tile_ids): ------- tile_list: [(tile_id, tile_data),...] A list of tile_id, tile_data tuples - ''' + """ generated_tiles = [] for tile_id in tile_ids: - tile_id_parts = tile_id.split('.') + tile_id_parts = tile_id.split(".") tile_position = list(map(int, tile_id_parts[1:3])) dense = hdft.get_data( - h5py.File( - tileset.datafile.path - ), - tile_position[0], - tile_position[1] + h5py.File(tileset.datafile.path), tile_position[0], tile_position[1] ) if len(dense): @@ -234,32 +244,35 @@ def generate_hitile_tiles(tileset, tile_ids): max_dense = 0 min_dense = 0 - min_f16 = np.finfo('float16').min - max_f16 = np.finfo('float16').max + min_f16 = np.finfo("float16").min + max_f16 = np.finfo("float16").max has_nan = len([d for d in dense if np.isnan(d)]) > 0 if ( - not has_nan and - max_dense > min_f16 and max_dense < max_f16 and - min_dense > min_f16 and min_dense < max_f16 + not has_nan + and max_dense > min_f16 + and max_dense < max_f16 + and min_dense > min_f16 + and min_dense < max_f16 ): tile_value = { - 'dense': base64.b64encode(dense.astype('float16')).decode('utf-8'), - 'dtype': 'float16' + "dense": base64.b64encode(dense.astype("float16")).decode("utf-8"), + "dtype": "float16", } else: tile_value = { - 'dense': base64.b64encode(dense.astype('float32')).decode('utf-8'), - 'dtype': 'float32' + "dense": base64.b64encode(dense.astype("float32")).decode("utf-8"), + "dtype": "float32", } generated_tiles += [(tile_id, tile_value)] return generated_tiles + def generate_bed2ddb_tiles(tileset, tile_ids, retriever=cdt.get_2d_tiles): - ''' + """ Generate tiles from a bed2db file. Parameters @@ -274,18 +287,19 @@ def generate_bed2ddb_tiles(tileset, tile_ids, retriever=cdt.get_2d_tiles): ------- generated_tiles: [(tile_id, tile_data),...] A list of tile_id, tile_data tuples - ''' + """ generated_tiles = [] tile_ids_by_zoom = bin_tiles_by_zoom(tile_ids).values() - partitioned_tile_ids = list(it.chain(*[partition_by_adjacent_tiles(t) - for t in tile_ids_by_zoom])) + partitioned_tile_ids = list( + it.chain(*[partition_by_adjacent_tiles(t) for t in tile_ids_by_zoom]) + ) for tile_group in partitioned_tile_ids: - zoom_level = int(tile_group[0].split('.')[1]) - tileset_id = tile_group[0].split('.')[0] + zoom_level = int(tile_group[0].split(".")[1]) + tileset_id = tile_group[0].split(".")[0] - tile_positions = [[int(x) for x in t.split('.')[2:4]] for t in tile_group] + tile_positions = [[int(x) for x in t.split(".")[2:4]] for t in tile_group] # filter for tiles that are in bounds for this zoom level tile_positions = list(filter(lambda x: x[0] < 2 ** zoom_level, tile_positions)) @@ -303,22 +317,24 @@ def generate_bed2ddb_tiles(tileset, tile_ids, retriever=cdt.get_2d_tiles): cached_datapath = get_cached_datapath(tileset.datafile.path) tile_data_by_position = retriever( - cached_datapath, - zoom_level, - minx, miny, - maxx - minx + 1, - maxy - miny + 1 - ) + cached_datapath, zoom_level, minx, miny, maxx - minx + 1, maxy - miny + 1 + ) - tiles = [(".".join(map(str, [tileset_id] + [zoom_level] + list(position))), tile_data) - for (position, tile_data) in tile_data_by_position.items()] + tiles = [ + ( + ".".join(map(str, [tileset_id] + [zoom_level] + list(position))), + tile_data, + ) + for (position, tile_data) in tile_data_by_position.items() + ] generated_tiles += tiles return generated_tiles + def generate_hibed_tiles(tileset, tile_ids): - ''' + """ Generate tiles from a hibed file. Parameters @@ -333,27 +349,26 @@ def generate_hibed_tiles(tileset, tile_ids): ------- generated_tiles: [(tile_id, tile_data),...] A list of tile_id, tile_data tuples - ''' + """ generated_tiles = [] for tile_id in tile_ids: - tile_id_parts = tile_id.split('.') + tile_id_parts = tile_id.split(".") tile_position = list(map(int, tile_id_parts[1:3])) dense = hdft.get_discrete_data( - h5py.File( - tileset.datafile.path - ), - tile_position[0], - tile_position[1] + h5py.File(tileset.datafile.path), tile_position[0], tile_position[1] ) - tile_value = {'discrete': list([list([x.decode('utf-8') for x in d]) for d in dense])} + tile_value = { + "discrete": list([list([x.decode("utf-8") for x in d]) for d in dense]) + } generated_tiles += [(tile_id, tile_value)] return generated_tiles + def bin_tiles_by_zoom(tile_ids): - ''' + """ Place these tiles into separate lists according to their zoom level. @@ -367,11 +382,11 @@ def bin_tiles_by_zoom(tile_ids): ------- tile_lists: {zoomLevel: [tile_id, tile_id]} A dictionary of tile lists - ''' + """ tile_id_lists = col.defaultdict(set) for tile_id in tile_ids: - tile_id_parts = tile_id.split('.') + tile_id_parts = tile_id.split(".") tile_position = list(map(int, tile_id_parts[1:4])) zoom_level = tile_position[0] @@ -381,7 +396,7 @@ def bin_tiles_by_zoom(tile_ids): def bin_tiles_by_zoom_level_and_transform(tile_ids): - ''' + """ Place these tiles into separate lists according to their zoom level and transform type @@ -395,11 +410,11 @@ def bin_tiles_by_zoom_level_and_transform(tile_ids): ------- tile_lists: {(zoomLevel, transformType): [tile_id, tile_id]} A dictionary of tile ids - ''' + """ tile_id_lists = col.defaultdict(set) for tile_id in tile_ids: - tile_id_parts = tile_id.split('.') + tile_id_parts = tile_id.split(".") tile_position = list(map(int, tile_id_parts[1:4])) zoom_level = tile_position[0] @@ -409,8 +424,9 @@ def bin_tiles_by_zoom_level_and_transform(tile_ids): return tile_id_lists + def partition_by_adjacent_tiles(tile_ids, dimension=2): - ''' + """ Partition a set of tile ids into sets of adjacent tiles Parameters @@ -426,11 +442,13 @@ def partition_by_adjacent_tiles(tile_ids, dimension=2): tile_lists: [tile_ids, tile_ids] A list of tile lists, all of which have tiles that are within 1 position of another tile in the list - ''' + """ tile_id_lists = [] - for tile_id in sorted(tile_ids, key=lambda x: [int(p) for p in x.split('.')[2:2+dimension]]): - tile_id_parts = tile_id.split('.') + for tile_id in sorted( + tile_ids, key=lambda x: [int(p) for p in x.split(".")[2 : 2 + dimension]] + ): + tile_id_parts = tile_id.split(".") # exclude the zoom level in the position # because the tiles should already have been partitioned @@ -444,12 +462,12 @@ def partition_by_adjacent_tiles(tile_ids, dimension=2): has_close_tile = False for ct_tile_id in tile_id_list: - ct_tile_id_parts = ct_tile_id.split('.') - ct_tile_position = list(map(int, ct_tile_id_parts[2:2+dimension])) + ct_tile_id_parts = ct_tile_id.split(".") + ct_tile_position = list(map(int, ct_tile_id_parts[2 : 2 + dimension])) far_apart = False # iterate over each dimension and see if this tile is close - for p1,p2 in zip(tile_position, ct_tile_position): + for p1, p2 in zip(tile_position, ct_tile_position): if abs(int(p1) - int(p2)) > 1: # too far apart can't be part of the same group far_apart = True @@ -467,8 +485,9 @@ def partition_by_adjacent_tiles(tile_ids, dimension=2): return tile_id_lists + def generate_tiles(tileset_tile_ids): - ''' + """ Generate a tiles for the give tile_ids. All of the tile_ids must come from the same tileset. This function @@ -487,40 +506,37 @@ def generate_tiles(tileset_tile_ids): ------- tile_list: [(tile_id, tile_data),...] A list of tile_id, tile_data tuples - ''' + """ tileset, tile_ids, raw = tileset_tile_ids - if tileset.filetype == 'hitile': + if tileset.filetype == "hitile": return generate_hitile_tiles(tileset, tile_ids) - elif tileset.filetype == 'beddb': + elif tileset.filetype == "beddb": return hgbe.tiles(tileset.datafile.path, tile_ids) - elif tileset.filetype == 'bed2ddb' or tileset.filetype == '2dannodb': + elif tileset.filetype == "bed2ddb" or tileset.filetype == "2dannodb": return generate_bed2ddb_tiles(tileset, tile_ids) - elif tileset.filetype == 'geodb': + elif tileset.filetype == "geodb": return generate_bed2ddb_tiles(tileset, tile_ids, hggo.get_tiles) - elif tileset.filetype == 'hibed': + elif tileset.filetype == "hibed": return generate_hibed_tiles(tileset, tile_ids) - elif tileset.filetype == 'cooler': + elif tileset.filetype == "cooler": return hgco.generate_tiles(tileset.datafile.path, tile_ids) - elif tileset.filetype == 'bigwig': + elif tileset.filetype == "bigwig": chromsizes = get_chromsizes(tileset) return hgbi.tiles(tileset.datafile.path, tile_ids, chromsizes=chromsizes) - elif tileset.filetype == 'bigbed': + elif tileset.filetype == "bigbed": chromsizes = get_chromsizes(tileset) return hgbb.tiles(tileset.datafile.path, tile_ids, chromsizes=chromsizes) - elif tileset.filetype == 'multivec': - return generate_1d_tiles( - tileset.datafile.path, - tile_ids, - ctmu.get_single_tile) - elif tileset.filetype == 'imtiles': + elif tileset.filetype == "multivec": + return generate_1d_tiles(tileset.datafile.path, tile_ids, ctmu.get_single_tile) + elif tileset.filetype == "imtiles": return hgim.get_tiles(tileset.datafile.path, tile_ids, raw) - elif tileset.filetype == 'bam': + elif tileset.filetype == "bam": return ctb.tiles( tileset.datafile.path, tile_ids, index_filename=tileset.indexfile.path, - max_tile_width=hss.MAX_BAM_TILE_WIDTH + max_tile_width=hss.MAX_BAM_TILE_WIDTH, ) else: filetype = tileset.filetype @@ -529,6 +545,7 @@ def generate_tiles(tileset_tile_ids): if filetype in hgti.by_filetype: return hgti.by_filetype[filetype](filepath).tiles(tile_ids) - return [(ti, {'error': 'Unknown tileset filetype: {}'.format(tileset.filetype)}) for ti in tile_ids] - - + return [ + (ti, {"error": "Unknown tileset filetype: {}".format(tileset.filetype)}) + for ti in tile_ids + ] diff --git a/tilesets/management/commands/delete_tileset.py b/tilesets/management/commands/delete_tileset.py index 8b6988b0..458b52ea 100644 --- a/tilesets/management/commands/delete_tileset.py +++ b/tilesets/management/commands/delete_tileset.py @@ -4,27 +4,32 @@ import tilesets.models as tm import os + class Command(BaseCommand): def add_arguments(self, parser): - parser.add_argument('--uuid', type=str, required=True) - + parser.add_argument("--uuid", type=str, required=True) + def handle(self, *args, **options): - uuid = options.get('uuid') - + uuid = options.get("uuid") + # search for Django object, remove associated file and record instance = tm.Tileset.objects.get(uuid=uuid) if not instance: - raise CommandError('Instance for specified uuid ({}) was not found'.format(uuid)) + raise CommandError( + "Instance for specified uuid ({}) was not found".format(uuid) + ) else: filename = instance.datafile.name filepath = os.path.join(settings.MEDIA_ROOT, filename) if not os.path.isfile(filepath): - raise CommandError('File does not exist under media root') + raise CommandError("File does not exist under media root") try: os.remove(filepath) except OSError: - raise CommandError('File under media root could not be removed') + raise CommandError("File under media root could not be removed") try: instance.delete() except ProtectedError: - raise CommandError('Instance for specified uuid ({}) could not be deleted'.format(uuid)) \ No newline at end of file + raise CommandError( + "Instance for specified uuid ({}) could not be deleted".format(uuid) + ) diff --git a/tilesets/management/commands/ingest_tileset.py b/tilesets/management/commands/ingest_tileset.py index 825297d5..dc91c92f 100644 --- a/tilesets/management/commands/ingest_tileset.py +++ b/tilesets/management/commands/ingest_tileset.py @@ -9,46 +9,60 @@ import logging import os import os.path as op -import tilesets.chromsizes as tcs +import tilesets.chromsizes as tcs from django.conf import settings logger = logging.getLogger(__name__) + def remote_to_local(filename, no_upload): - if filename[:7] == 'http://': - filename = "{}..".format(filename.replace('http:/', 'http')) - no_upload=True - if filename[:8] == 'https://': - filename = "{}..".format(filename.replace('https:/', 'https')) - no_upload=True - if filename[:6] == 'ftp://': - filename = "{}..".format(filename.replace('ftp:/', 'ftp')) - no_upload=True + if filename[:7] == "http://": + filename = "{}..".format(filename.replace("http:/", "http")) + no_upload = True + if filename[:8] == "https://": + filename = "{}..".format(filename.replace("https:/", "https")) + no_upload = True + if filename[:6] == "ftp://": + filename = "{}..".format(filename.replace("ftp:/", "ftp")) + no_upload = True return (filename, no_upload) -def ingest(filename=None, datatype=None, filetype=None, coordSystem='', coordSystem2='', - uid=None, name=None, no_upload=False, project_name='', - indexfile=None, temporary=False, **ignored): + +def ingest( + filename=None, + datatype=None, + filetype=None, + coordSystem="", + coordSystem2="", + uid=None, + name=None, + no_upload=False, + project_name="", + indexfile=None, + temporary=False, + **ignored, +): uid = uid or slugid.nice() name = name or op.split(filename)[1] if not filetype: - raise CommandError('Filetype has to be specified') + raise CommandError("Filetype has to be specified") django_file = None # bamfiles need to be ingested with an index, if it's not # specified as a parameter, try to find it at filename + ".bai" # and complain if it can't be found - if filetype == 'bam': + if filetype == "bam": if indexfile is None: - indexfile = filename + '.bai' + indexfile = filename + ".bai" if not op.exists(indexfile): - print(f'Index for bamfile {indexfile} not found. '+ - 'Either specify explicitly using the --indexfile parameter ' + - 'or create it in the expected location.') - return + raise CommandError( + f"Index for bamfile {indexfile} not found. " + + "Either specify explicitly using the --indexfile parameter " + + "or create it in the expected location." + ) # if we're ingesting a url, place it relative to the httpfs directories # and append two dots at the end @@ -59,42 +73,56 @@ def ingest(filename=None, datatype=None, filetype=None, coordSystem='', coordSys # it's a regular file on the filesystem, not a file being entered as a url if no_upload: - if (not op.isfile(op.join(settings.MEDIA_ROOT, filename)) and - not op.islink(op.join(settings.MEDIA_ROOT, filename)) and - not any([filename.startswith('http/'), filename.startswith('https/'), filename.startswith('ftp/')])): - raise CommandError('File does not exist under media root') + if ( + not op.isfile(op.join(settings.MEDIA_ROOT, filename)) + and not op.islink(op.join(settings.MEDIA_ROOT, filename)) + and not any( + [ + filename.startswith("http/"), + filename.startswith("https/"), + filename.startswith("ftp/"), + ] + ) + ): + raise CommandError("File does not exist under media root") filename = op.join(settings.MEDIA_ROOT, filename) django_file = filename if indexfile: - if (not op.isfile(op.join(settings.MEDIA_ROOT, indexfile)) and - not op.islink(op.join(settings.MEDIA_ROOT, indexfile)) and - not any([indexfile.startswith('http/'), indexfile.startswith('https/'), indexfile.startswith('ftp/')])): - raise CommandError('Index file does not exist under media root') + if ( + not op.isfile(op.join(settings.MEDIA_ROOT, indexfile)) + and not op.islink(op.join(settings.MEDIA_ROOT, indexfile)) + and not any( + [ + indexfile.startswith("http/"), + indexfile.startswith("https/"), + indexfile.startswith("ftp/"), + ] + ) + ): + raise CommandError("Index file does not exist under media root") indexfile = op.join(settings.MEDIA_ROOT, indexfile) else: if os.path.islink(filename): - django_file = File(open(os.readlink(filename),'rb')) + django_file = File(open(os.readlink(filename), "rb")) if indexfile: - indexfile = File(open(os.readlink(indexfile, 'rb'))) + indexfile = File(open(os.readlink(indexfile, "rb"))) else: - django_file = File(open(filename,'rb')) + django_file = File(open(filename, "rb")) if indexfile: - indexfile = File(open(indexfile, 'rb')) + indexfile = File(open(indexfile, "rb")) # remove the filepath of the filename django_file.name = op.split(django_file.name)[1] if indexfile: indexfile.name = op.split(indexfile.name)[1] - if filetype.lower() == 'bigwig' or filetype.lower() == 'bigbed': + if filetype.lower() == "bigwig" or filetype.lower() == "bigbed": coordSystem = check_for_chromsizes(filename, coordSystem) try: project_obj = tm.Project.objects.get(name=project_name) except dce.ObjectDoesNotExist: - project_obj = tm.Project.objects.create( - name=project_name - ) + project_obj = tm.Project.objects.create(name=project_name) return tm.Tileset.objects.create( datafile=django_file, @@ -107,13 +135,16 @@ def ingest(filename=None, datatype=None, filetype=None, coordSystem='', coordSys project=project_obj, uuid=uid, temporary=temporary, - name=name) + name=name, + ) + def chromsizes_match(chromsizes1, chromsizes2): pass + def check_for_chromsizes(filename, coord_system): - ''' + """ Check to see if we have chromsizes matching the coord system of the filename. @@ -123,10 +154,12 @@ def check_for_chromsizes(filename, coord_system): The name of the bigwig file coord_system: string The coordinate system (assembly) of this bigwig file - ''' + """ tileset_info = hgbi.tileset_info(filename) # print("tileset chromsizes:", tileset_info['chromsizes']) - tsinfo_chromsizes = set([(str(chrom), str(size)) for chrom, size in tileset_info['chromsizes']]) + tsinfo_chromsizes = set( + [(str(chrom), str(size)) for chrom, size in tileset_info["chromsizes"]] + ) # print("tsinfo_chromsizes:", tsinfo_chromsizes) chrom_info_tileset = None @@ -136,12 +169,14 @@ def check_for_chromsizes(filename, coord_system): if coord_system is not None and len(coord_system) > 0: try: chrom_info_tileset = tm.Tileset.objects.filter( - coordSystem=coord_system, - datatype='chromsizes') + coordSystem=coord_system, datatype="chromsizes" + ) if len(chrom_info_tileset) > 1: - raise CommandError("More than one available set of chromSizes" - + "for this coordSystem ({})".format(coord_system)) + raise CommandError( + "More than one available set of chromSizes" + + "for this coordSystem ({})".format(coord_system) + ) chrom_info_tileset = chrom_info_tileset.first() except dce.ObjectDoesNotExist: @@ -152,59 +187,85 @@ def check_for_chromsizes(filename, coord_system): if chrom_info_tileset is None: # we haven't found chromsizes matching the coordsystem # go through every chromsizes file and see if we have a match - for chrom_info_tileset in tm.Tileset.objects.filter(datatype='chromsizes'): - chromsizes_set = set([tuple(t) for - t in tcs.get_tsv_chromsizes(chrom_info_tileset.datafile.path)]) - - matches += [(len(set.intersection(chromsizes_set, tsinfo_chromsizes)), - chrom_info_tileset)] + for chrom_info_tileset in tm.Tileset.objects.filter(datatype="chromsizes"): + chromsizes_set = set( + [ + tuple(t) + for t in tcs.get_tsv_chromsizes(chrom_info_tileset.datafile.path) + ] + ) + + matches += [ + ( + len(set.intersection(chromsizes_set, tsinfo_chromsizes)), + chrom_info_tileset, + ) + ] # print("chrom_info_tileset:", chromsizes_set) - #print("intersection:", len(set.intersection(chromsizes_set, tsinfo_chromsizes))) - #print("coord_system:", coord_system) + # print("intersection:", len(set.intersection(chromsizes_set, tsinfo_chromsizes))) + # print("coord_system:", coord_system) else: # a set of chromsizes was provided - chromsizes_set = set([tuple(t) for - t in tcs.get_tsv_chromsizes(chrom_info_tileset.datafile.path)]) - matches += [(len(set.intersection(chromsizes_set, tsinfo_chromsizes)), - chrom_info_tileset)] + chromsizes_set = set( + [tuple(t) for t in tcs.get_tsv_chromsizes(chrom_info_tileset.datafile.path)] + ) + matches += [ + ( + len(set.intersection(chromsizes_set, tsinfo_chromsizes)), + chrom_info_tileset, + ) + ] # matches that overlap some chromsizes with the bigwig file overlap_matches = [m for m in matches if m[0] > 0] if len(overlap_matches) == 0: - raise CommandError("No chromsizes available which match the chromosomes in this bigwig" - + "See http://docs.higlass.io/data_preparation.html#bigwig-files " - + "for more information" - ) + raise CommandError( + "No chromsizes available which match the chromosomes in this bigwig" + + "See http://docs.higlass.io/data_preparation.html#bigwig-files " + + "for more information" + ) if len(overlap_matches) > 1: - raise CommandError("Multiple matching coordSystems:" - + "See http://docs.higlass.io/data_preparation.html#bigwig-files " - + "for more information", - ["({} [{}])".format(t[1].coordSystem, t[0]) for t in overlap_matches]) - - if (coord_system is not None - and len(coord_system) > 0 - and overlap_matches[0][1].coordSystem != coord_system): - raise CommandError("Matching chromosome sizes (coordSystem: {}) do not " - + "match the specified coordinate sytem ({}). " + raise CommandError( + "Multiple matching coordSystems:" + + "See http://docs.higlass.io/data_preparation.html#bigwig-files " + + "for more information", + ["({} [{}])".format(t[1].coordSystem, t[0]) for t in overlap_matches], + ) + + if ( + coord_system is not None + and len(coord_system) > 0 + and overlap_matches[0][1].coordSystem != coord_system + ): + raise CommandError( + f"Matching chromosome sizes (coordSystem: {overlap_matches[0][1].coordSystem}) do not " + + f"match the specified coordinate sytem ({coord_system}). " + "Either omit the coordSystem or specify a matching one." + "See http://docs.higlass.io/data_preparation.html#bigwig-files " - + "for more information".format(overlap_matches[0][1].coordSystem, coord_system)) + + "for more information" + ) - if (coord_system is not None - and len(coord_system) > 0 - and overlap_matches[0][1].coordSystem == coord_system): + if ( + coord_system is not None + and len(coord_system) > 0 + and overlap_matches[0][1].coordSystem == coord_system + ): print("Using coordinates for coordinate system: {}".format(coord_system)) if coord_system is None or len(coord_system) == 0: - print("No coordinate system specified, but we found matching " - + "chromsizes. Using coordinate system {}." - .format(overlap_matches[0][1].coordSystem)) + print( + "No coordinate system specified, but we found matching " + + "chromsizes. Using coordinate system {}.".format( + overlap_matches[0][1].coordSystem + ) + ) return overlap_matches[0][1].coordSystem + class Command(BaseCommand): def add_arguments(self, parser): # TODO: filename, datatype, fileType and coordSystem should @@ -212,24 +273,24 @@ def add_arguments(self, parser): # for now, coordSystem2 should take the value of coordSystem # if the datatype is matrix # otherwise, coordSystem2 should be empty - parser.add_argument('--filename', type=str) - parser.add_argument('--indexfile', type=str) - parser.add_argument('--datatype', type=str) - parser.add_argument('--filetype', type=str) - parser.add_argument('--coordSystem', default='', type=str) - parser.add_argument('--coordSystem2', default='', type=str) + parser.add_argument("--filename", type=str) + parser.add_argument("--indexfile", type=str) + parser.add_argument("--datatype", type=str) + parser.add_argument("--filetype", type=str) + parser.add_argument("--coordSystem", default="", type=str) + parser.add_argument("--coordSystem2", default="", type=str) # parser.add_argument('--coord', default='hg19', type=str) - parser.add_argument('--uid', type=str) - parser.add_argument('--name', type=str) - parser.add_argument('--project-name', type=str, default='') + parser.add_argument("--uid", type=str) + parser.add_argument("--name", type=str) + parser.add_argument("--project-name", type=str, default="") # Named (optional) arguments parser.add_argument( - '--no-upload', - action='store_true', - dest='no_upload', + "--no-upload", + action="store_true", + dest="no_upload", default=False, - help='Skip upload', + help="Skip upload", ) def handle(self, *args, **options): diff --git a/tilesets/management/commands/list_tilesets.py b/tilesets/management/commands/list_tilesets.py index 9a2efadc..18319756 100644 --- a/tilesets/management/commands/list_tilesets.py +++ b/tilesets/management/commands/list_tilesets.py @@ -13,4 +13,4 @@ def add_arguments(self, parser): def handle(self, *args, **options): for tileset in tm.Tileset.objects.all(): - print('tileset:', tileset) + print("tileset:", tileset) diff --git a/tilesets/management/commands/modify_tileset.py b/tilesets/management/commands/modify_tileset.py index fb685aa1..3b174612 100644 --- a/tilesets/management/commands/modify_tileset.py +++ b/tilesets/management/commands/modify_tileset.py @@ -4,31 +4,38 @@ import tilesets.models as tm import os + class Command(BaseCommand): def add_arguments(self, parser): - parser.add_argument('--uuid', type=str, required=True) - parser.add_argument('--name', type=str) - + parser.add_argument("--uuid", type=str, required=True) + parser.add_argument("--name", type=str) + def handle(self, *args, **options): - uuid = options.get('uuid') - name = options.get('name') - + uuid = options.get("uuid") + name = options.get("name") + # search for Django object, modify associated record instance = tm.Tileset.objects.get(uuid=uuid) if not instance: - raise CommandError('Instance for specified uuid ({}) was not found'.format(uuid)) + raise CommandError( + "Instance for specified uuid ({}) was not found".format(uuid) + ) else: try: instance_dirty = False - - # only change tileset name if specified, and if it is + + # only change tileset name if specified, and if it is # different from the current instance name if name and name != instance.name: instance.name = name instance_dirty = True - + # if any changes were applied, persist them if instance_dirty: instance.save() except ProtectedError: - raise CommandError('Instance for specified uuid ({}) could not be modified'.format(uuid)) \ No newline at end of file + raise CommandError( + "Instance for specified uuid ({}) could not be modified".format( + uuid + ) + ) diff --git a/tilesets/migrations/0001_initial.py b/tilesets/migrations/0001_initial.py index 907e4c83..165dd69d 100644 --- a/tilesets/migrations/0001_initial.py +++ b/tilesets/migrations/0001_initial.py @@ -12,41 +12,70 @@ class Migration(migrations.Migration): initial = True - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] + dependencies = [migrations.swappable_dependency(settings.AUTH_USER_MODEL)] operations = [ migrations.CreateModel( - name='Tileset', + name="Tileset", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created', models.DateTimeField(auto_now_add=True)), - ('uuid', models.CharField(default=slugid.slugid.nice, max_length=100, unique=True)), - ('datafile', models.FileField(upload_to='uploads')), - ('filetype', models.TextField()), - ('datatype', models.TextField(default='unknown')), - ('coordSystem', models.TextField()), - ('coordSystem2', models.TextField(default='')), - ('private', models.BooleanField(default=False)), - ('name', models.TextField(blank=True)), - ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tilesets', to=settings.AUTH_USER_MODEL)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ( + "uuid", + models.CharField( + default=slugid.slugid.nice, max_length=100, unique=True + ), + ), + ("datafile", models.FileField(upload_to="uploads")), + ("filetype", models.TextField()), + ("datatype", models.TextField(default="unknown")), + ("coordSystem", models.TextField()), + ("coordSystem2", models.TextField(default="")), + ("private", models.BooleanField(default=False)), + ("name", models.TextField(blank=True)), + ( + "owner", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="tilesets", + to=settings.AUTH_USER_MODEL, + ), + ), ], options={ - 'ordering': ('created',), - 'permissions': (('view_tileset', 'View tileset'),), + "ordering": ("created",), + "permissions": (("view_tileset", "View tileset"),), }, ), migrations.CreateModel( - name='ViewConf', + name="ViewConf", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created', models.DateTimeField(auto_now_add=True)), - ('uuid', models.CharField(default=slugid.slugid.nice, max_length=100, unique=True)), - ('viewconf', models.TextField()), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ( + "uuid", + models.CharField( + default=slugid.slugid.nice, max_length=100, unique=True + ), + ), + ("viewconf", models.TextField()), ], - options={ - 'ordering': ('created',), - }, + options={"ordering": ("created",)}, ), ] diff --git a/tilesets/migrations/0002_auto_20170223_1629.py b/tilesets/migrations/0002_auto_20170223_1629.py index 28d92797..5edfacee 100644 --- a/tilesets/migrations/0002_auto_20170223_1629.py +++ b/tilesets/migrations/0002_auto_20170223_1629.py @@ -9,14 +9,18 @@ class Migration(migrations.Migration): - dependencies = [ - ('tilesets', '0001_initial'), - ] + dependencies = [("tilesets", "0001_initial")] operations = [ migrations.AlterField( - model_name='tileset', - name='owner', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='tilesets', to=settings.AUTH_USER_MODEL), - ), + model_name="tileset", + name="owner", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="tilesets", + to=settings.AUTH_USER_MODEL, + ), + ) ] diff --git a/tilesets/migrations/0003_viewconf_higlassversion.py b/tilesets/migrations/0003_viewconf_higlassversion.py index d0f001d0..83898eaf 100644 --- a/tilesets/migrations/0003_viewconf_higlassversion.py +++ b/tilesets/migrations/0003_viewconf_higlassversion.py @@ -7,14 +7,12 @@ class Migration(migrations.Migration): - dependencies = [ - ('tilesets', '0002_auto_20170223_1629'), - ] + dependencies = [("tilesets", "0002_auto_20170223_1629")] operations = [ migrations.AddField( - model_name='viewconf', - name='higlassVersion', - field=models.CharField(default='', max_length=5), - ), + model_name="viewconf", + name="higlassVersion", + field=models.CharField(default="", max_length=5), + ) ] diff --git a/tilesets/migrations/0004_auto_20181115_1744.py b/tilesets/migrations/0004_auto_20181115_1744.py index 8d730cbd..05b65c9e 100644 --- a/tilesets/migrations/0004_auto_20181115_1744.py +++ b/tilesets/migrations/0004_auto_20181115_1744.py @@ -5,19 +5,19 @@ class Migration(migrations.Migration): - dependencies = [ - ('tilesets', '0003_viewconf_higlassversion'), - ] + dependencies = [("tilesets", "0003_viewconf_higlassversion")] operations = [ migrations.AlterField( - model_name='tileset', - name='uuid', - field=models.CharField(default='asXQG1k1Rwe1sqzkuvEtvw', max_length=100, unique=True), + model_name="tileset", + name="uuid", + field=models.CharField( + default="asXQG1k1Rwe1sqzkuvEtvw", max_length=100, unique=True + ), ), migrations.AlterField( - model_name='viewconf', - name='higlassVersion', - field=models.CharField(default='', max_length=16), + model_name="viewconf", + name="higlassVersion", + field=models.CharField(default="", max_length=16), ), ] diff --git a/tilesets/migrations/0005_auto_20181127_0239.py b/tilesets/migrations/0005_auto_20181127_0239.py index 7f857434..b11f66c9 100644 --- a/tilesets/migrations/0005_auto_20181127_0239.py +++ b/tilesets/migrations/0005_auto_20181127_0239.py @@ -11,54 +11,106 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('tilesets', '0004_auto_20181115_1744'), + ("tilesets", "0004_auto_20181115_1744"), ] operations = [ migrations.CreateModel( - name='Project', + name="Project", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created', models.DateTimeField(auto_now_add=True)), - ('last_viewed_time', models.DateTimeField(default=django.utils.timezone.now)), - ('name', models.TextField()), - ('description', models.TextField(blank=True)), - ('uuid', models.CharField(default=tilesets.models.decoded_slugid, max_length=100, unique=True)), - ('private', models.BooleanField(default=False)), - ('owner', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ( + "last_viewed_time", + models.DateTimeField(default=django.utils.timezone.now), + ), + ("name", models.TextField()), + ("description", models.TextField(blank=True)), + ( + "uuid", + models.CharField( + default=tilesets.models.decoded_slugid, + max_length=100, + unique=True, + ), + ), + ("private", models.BooleanField(default=False)), + ( + "owner", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), ], options={ - 'ordering': ('created',), - 'permissions': (('read', 'Read permission'), ('write', 'Modify tileset'), ('admin', 'Administrator priviliges')), + "ordering": ("created",), + "permissions": ( + ("read", "Read permission"), + ("write", "Modify tileset"), + ("admin", "Administrator priviliges"), + ), }, ), migrations.CreateModel( - name='Tag', + name="Tag", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created', models.DateTimeField(auto_now_add=True)), - ('name', models.TextField(unique=True)), - ('description', models.TextField(blank=True, default='')), - ('refs', models.IntegerField(default=0)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("name", models.TextField(unique=True)), + ("description", models.TextField(blank=True, default="")), + ("refs", models.IntegerField(default=0)), ], ), migrations.AlterModelOptions( - name='tileset', - options={'ordering': ('created',), 'permissions': (('read', 'Read permission'), ('write', 'Modify tileset'), ('admin', 'Administrator priviliges'))}, + name="tileset", + options={ + "ordering": ("created",), + "permissions": ( + ("read", "Read permission"), + ("write", "Modify tileset"), + ("admin", "Administrator priviliges"), + ), + }, ), migrations.AlterField( - model_name='tileset', - name='uuid', - field=models.CharField(default=tilesets.models.decoded_slugid, max_length=100, unique=True), + model_name="tileset", + name="uuid", + field=models.CharField( + default=tilesets.models.decoded_slugid, max_length=100, unique=True + ), ), migrations.AddField( - model_name='tileset', - name='project', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='tilesets.Project'), + model_name="tileset", + name="project", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="tilesets.Project", + ), ), migrations.AddField( - model_name='tileset', - name='tags', - field=models.ManyToManyField(blank=True, to='tilesets.Tag'), + model_name="tileset", + name="tags", + field=models.ManyToManyField(blank=True, to="tilesets.Tag"), ), ] diff --git a/tilesets/migrations/0006_tileset_description.py b/tilesets/migrations/0006_tileset_description.py index aea849a3..920b4b98 100644 --- a/tilesets/migrations/0006_tileset_description.py +++ b/tilesets/migrations/0006_tileset_description.py @@ -5,14 +5,10 @@ class Migration(migrations.Migration): - dependencies = [ - ('tilesets', '0005_auto_20181127_0239'), - ] + dependencies = [("tilesets", "0005_auto_20181127_0239")] operations = [ migrations.AddField( - model_name='tileset', - name='description', - field=models.TextField(blank=True), - ), + model_name="tileset", name="description", field=models.TextField(blank=True) + ) ] diff --git a/tilesets/migrations/0007_auto_20181127_0254.py b/tilesets/migrations/0007_auto_20181127_0254.py index 391fabb0..3813f407 100644 --- a/tilesets/migrations/0007_auto_20181127_0254.py +++ b/tilesets/migrations/0007_auto_20181127_0254.py @@ -5,14 +5,10 @@ class Migration(migrations.Migration): - dependencies = [ - ('tilesets', '0006_tileset_description'), - ] + dependencies = [("tilesets", "0006_tileset_description")] operations = [ migrations.AlterField( - model_name='project', - name='name', - field=models.TextField(unique=True), - ), + model_name="project", name="name", field=models.TextField(unique=True) + ) ] diff --git a/tilesets/migrations/0008_auto_20181129_1304.py b/tilesets/migrations/0008_auto_20181129_1304.py index 1a4025c8..c94c55b9 100644 --- a/tilesets/migrations/0008_auto_20181129_1304.py +++ b/tilesets/migrations/0008_auto_20181129_1304.py @@ -5,16 +5,9 @@ class Migration(migrations.Migration): - dependencies = [ - ('tilesets', '0007_auto_20181127_0254'), - ] + dependencies = [("tilesets", "0007_auto_20181127_0254")] operations = [ - migrations.RemoveField( - model_name='tileset', - name='tags', - ), - migrations.DeleteModel( - name='Tag', - ), + migrations.RemoveField(model_name="tileset", name="tags"), + migrations.DeleteModel(name="Tag"), ] diff --git a/tilesets/migrations/0009_tileset_temporary.py b/tilesets/migrations/0009_tileset_temporary.py index eef357d6..f9961df6 100644 --- a/tilesets/migrations/0009_tileset_temporary.py +++ b/tilesets/migrations/0009_tileset_temporary.py @@ -5,14 +5,12 @@ class Migration(migrations.Migration): - dependencies = [ - ('tilesets', '0008_auto_20181129_1304'), - ] + dependencies = [("tilesets", "0008_auto_20181129_1304")] operations = [ migrations.AddField( - model_name='tileset', - name='temporary', + model_name="tileset", + name="temporary", field=models.BooleanField(default=False), - ), + ) ] diff --git a/tilesets/migrations/0010_auto_20181228_2250.py b/tilesets/migrations/0010_auto_20181228_2250.py index 260d754b..b4c5bb3a 100644 --- a/tilesets/migrations/0010_auto_20181228_2250.py +++ b/tilesets/migrations/0010_auto_20181228_2250.py @@ -5,14 +5,12 @@ class Migration(migrations.Migration): - dependencies = [ - ('tilesets', '0009_tileset_temporary'), - ] + dependencies = [("tilesets", "0009_tileset_temporary")] operations = [ migrations.AlterField( - model_name='tileset', - name='datatype', - field=models.TextField(blank=True, default='unknown'), - ), + model_name="tileset", + name="datatype", + field=models.TextField(blank=True, default="unknown"), + ) ] diff --git a/tilesets/migrations/0011_auto_20181228_2252.py b/tilesets/migrations/0011_auto_20181228_2252.py index adbc44f9..69969e6b 100644 --- a/tilesets/migrations/0011_auto_20181228_2252.py +++ b/tilesets/migrations/0011_auto_20181228_2252.py @@ -5,14 +5,12 @@ class Migration(migrations.Migration): - dependencies = [ - ('tilesets', '0010_auto_20181228_2250'), - ] + dependencies = [("tilesets", "0010_auto_20181228_2250")] operations = [ migrations.AlterField( - model_name='tileset', - name='datatype', - field=models.TextField(blank=True, default='unknown', null=True), - ), + model_name="tileset", + name="datatype", + field=models.TextField(blank=True, default="unknown", null=True), + ) ] diff --git a/tilesets/migrations/0012_auto_20190923_0257.py b/tilesets/migrations/0012_auto_20190923_0257.py index 8b7c3f21..6721c6b8 100644 --- a/tilesets/migrations/0012_auto_20190923_0257.py +++ b/tilesets/migrations/0012_auto_20190923_0257.py @@ -5,19 +5,17 @@ class Migration(migrations.Migration): - dependencies = [ - ('tilesets', '0011_auto_20181228_2252'), - ] + dependencies = [("tilesets", "0011_auto_20181228_2252")] operations = [ migrations.AddField( - model_name='tileset', - name='indexfile', - field=models.FileField(default=None, null=True, upload_to='uploads'), + model_name="tileset", + name="indexfile", + field=models.FileField(default=None, null=True, upload_to="uploads"), ), migrations.AlterField( - model_name='tileset', - name='coordSystem2', - field=models.TextField(blank=True, default=''), + model_name="tileset", + name="coordSystem2", + field=models.TextField(blank=True, default=""), ), ] diff --git a/tilesets/models.py b/tilesets/models.py index 60e265af..5d9aa637 100644 --- a/tilesets/models.py +++ b/tilesets/models.py @@ -9,80 +9,93 @@ class ViewConf(models.Model): created = models.DateTimeField(auto_now_add=True) - higlassVersion = models.CharField(max_length=16, default='') + higlassVersion = models.CharField(max_length=16, default="") uuid = models.CharField(max_length=100, unique=True, default=slugid.nice) viewconf = models.TextField() class Meta: - ordering = ('created',) + ordering = ("created",) def __str__(self): - ''' + """ Get a string representation of this model. Hopefully useful for the admin interface. - ''' + """ return "Viewconf [uuid: {}]".format(self.uuid) + def decoded_slugid(): return slugid.nice() + class Project(models.Model): created = models.DateTimeField(auto_now_add=True) last_viewed_time = models.DateTimeField(default=django.utils.timezone.now) - owner = models.ForeignKey(dcam.User, on_delete=models.CASCADE, blank=True, null=True) + owner = models.ForeignKey( + dcam.User, on_delete=models.CASCADE, blank=True, null=True + ) name = models.TextField(unique=True) description = models.TextField(blank=True) uuid = models.CharField(max_length=100, unique=True, default=decoded_slugid) private = models.BooleanField(default=False) class Meta: - ordering = ('created',) - permissions = (('read', "Read permission"), - ('write', 'Modify tileset'), - ('admin', 'Administrator priviliges'), - ) + ordering = ("created",) + permissions = ( + ("read", "Read permission"), + ("write", "Modify tileset"), + ("admin", "Administrator priviliges"), + ) def __str__(self): return "Project [name: " + self.name + "]" + class Tileset(models.Model): created = models.DateTimeField(auto_now_add=True) uuid = models.CharField(max_length=100, unique=True, default=decoded_slugid) # processed_file = models.TextField() - datafile = models.FileField(upload_to='uploads') + datafile = models.FileField(upload_to="uploads") # indexfile is used for bam files - indexfile = models.FileField(upload_to='uploads', default=None, null=True) + indexfile = models.FileField(upload_to="uploads", default=None, null=True) filetype = models.TextField() - datatype = models.TextField(default='unknown', blank=True, null=True) - project = models.ForeignKey(Project, on_delete=models.CASCADE, - blank=True, null=True) + datatype = models.TextField(default="unknown", blank=True, null=True) + project = models.ForeignKey( + Project, on_delete=models.CASCADE, blank=True, null=True + ) description = models.TextField(blank=True) coordSystem = models.TextField() - coordSystem2 = models.TextField(default='', blank=True) + coordSystem2 = models.TextField(default="", blank=True) temporary = models.BooleanField(default=False) owner = models.ForeignKey( - 'auth.User', related_name='tilesets', on_delete=models.CASCADE, - blank=True, null=True # Allow anonymous owner + "auth.User", + related_name="tilesets", + on_delete=models.CASCADE, + blank=True, + null=True, # Allow anonymous owner ) private = models.BooleanField(default=False) name = models.TextField(blank=True) class Meta: - ordering = ('created',) - permissions = (('read', "Read permission"), - ('write', 'Modify tileset'), - ('admin', 'Administrator priviliges'), - ) + ordering = ("created",) + permissions = ( + ("read", "Read permission"), + ("write", "Modify tileset"), + ("admin", "Administrator priviliges"), + ) def __str__(self): - ''' + """ Get a string representation of this model. Hopefully useful for the admin interface. - ''' - return "Tileset [name: {}] [ft: {}] [uuid: {}]".format(self.name, self.filetype, self.uuid) + """ + return "Tileset [name: {}] [ft: {}] [uuid: {}]".format( + self.name, self.filetype, self.uuid + ) diff --git a/tilesets/permissions.py b/tilesets/permissions.py index 63b819e7..baecd5f3 100644 --- a/tilesets/permissions.py +++ b/tilesets/permissions.py @@ -5,7 +5,7 @@ class IsRequestMethodGet(permissions.BasePermission): """The request is a GET request.""" def has_permission(self, request, view): - if request.method == 'GET': + if request.method == "GET": return True return False @@ -29,22 +29,22 @@ class UserPermission(permissions.BasePermission): # Taken from http://stackoverflow.com/a/34162842/899470 def has_permission(self, request, view): - if view.action in ['retrieve', 'list']: + if view.action in ["retrieve", "list"]: return True - elif view.action in ['create', 'update', 'partial_update', 'destroy']: + elif view.action in ["create", "update", "partial_update", "destroy"]: return request.user.is_authenticated else: return False def has_object_permission(self, request, view, obj): - if view.action == 'retrieve': - return ( - request.user.is_authenticated and - (obj == request.user or request.user.is_superuser) + if view.action == "retrieve": + return request.user.is_authenticated and ( + obj == request.user or request.user.is_superuser ) - elif view.action in ['update', 'partial_update', 'destroy']: + elif view.action in ["update", "partial_update", "destroy"]: return request.user.is_authenticated and ( - request.user.is_superuser or request.user == obj.owner) + request.user.is_superuser or request.user == obj.owner + ) else: return False @@ -53,16 +53,15 @@ class UserPermissionReadOnly(UserPermission): """Custom permission to only allow read requests.""" def has_permission(self, request, view): - if view.action in ['retrieve', 'list']: + if view.action in ["retrieve", "list"]: return True else: return False def has_object_permission(self, request, view, obj): - if view.action == 'retrieve': - return ( - request.user.is_authenticated and - (obj == request.user or request.user.is_superuser) + if view.action == "retrieve": + return request.user.is_authenticated and ( + obj == request.user or request.user.is_superuser ) else: return False diff --git a/tilesets/serializers.py b/tilesets/serializers.py index 428ecb9b..4aac514b 100644 --- a/tilesets/serializers.py +++ b/tilesets/serializers.py @@ -10,15 +10,12 @@ logger = logging.getLogger(__name__) + class ProjectsSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = tm.Project - fields = ( - 'uuid', - 'name', - 'description', - 'private' - ) + fields = ("uuid", "name", "description", "private") + class UserSerializer(serializers.ModelSerializer): tilesets = serializers.PrimaryKeyRelatedField( @@ -27,81 +24,82 @@ class UserSerializer(serializers.ModelSerializer): class Meta: model = User - fields = ('id', 'username') + fields = ("id", "username") class ViewConfSerializer(serializers.ModelSerializer): class Meta: model = ViewConf - fields = ('uuid', 'viewconf') + fields = ("uuid", "viewconf") class TilesetSerializer(serializers.ModelSerializer): project = serializers.SlugRelatedField( - queryset=tm.Project.objects.all(), - slug_field='uuid', - allow_null=True, - required=False) - project_name = serializers.SerializerMethodField('retrieve_project_name') + queryset=tm.Project.objects.all(), + slug_field="uuid", + allow_null=True, + required=False, + ) + project_name = serializers.SerializerMethodField("retrieve_project_name") def retrieve_project_name(self, obj): if obj.project is None: - return '' + return "" return obj.project.name class Meta: - owner = serializers.ReadOnlyField(source='owner.username') + owner = serializers.ReadOnlyField(source="owner.username") model = tm.Tileset fields = ( - 'uuid', - 'datafile', - 'filetype', - 'datatype', - 'name', - 'coordSystem', - 'coordSystem2', - 'created', - 'project', - 'project_name', - 'description', - 'private', + "uuid", + "datafile", + "filetype", + "datatype", + "name", + "coordSystem", + "coordSystem2", + "created", + "project", + "project_name", + "description", + "private", ) class UserFacingTilesetSerializer(TilesetSerializer): - owner = serializers.ReadOnlyField(source='owner.username') - project_name = serializers.SerializerMethodField('retrieve_project_name') - project_owner = serializers.SerializerMethodField('retrieve_project_owner') + owner = serializers.ReadOnlyField(source="owner.username") + project_name = serializers.SerializerMethodField("retrieve_project_name") + project_owner = serializers.SerializerMethodField("retrieve_project_owner") def retrieve_project_name(self, obj): if obj.project is None: - return '' + return "" return obj.project.name def retrieve_project_owner(self, obj): if obj.project is None: - return '' + return "" if obj.project.owner is None: - return '' + return "" return obj.project.owner.username class Meta: model = tm.Tileset fields = ( - 'uuid', - 'filetype', - 'datatype', - 'private', - 'name', - 'coordSystem', - 'coordSystem2', - 'created', - 'owner', - 'project_name', - 'project_owner', - 'description', + "uuid", + "filetype", + "datatype", + "private", + "name", + "coordSystem", + "coordSystem2", + "created", + "owner", + "project_name", + "project_owner", + "description", ) diff --git a/tilesets/suggestions.py b/tilesets/suggestions.py index eb3e9a3d..aeeed69e 100644 --- a/tilesets/suggestions.py +++ b/tilesets/suggestions.py @@ -1,7 +1,8 @@ import sqlite3 + def get_gene_suggestions(db_file, text): - ''' + """ Get a list of autocomplete suggestions for genes containing the text. @@ -14,7 +15,7 @@ def get_gene_suggestions(db_file, text): suggestions (list): A list of dictionaries containing the suggestions: e.g. ([{'txStart': 10, 'txEnd': 20, 'score': 15, 'geneName': 'XV4'}]) - ''' + """ con = sqlite3.connect(db_file) c = con.cursor() @@ -23,22 +24,25 @@ def get_gene_suggestions(db_file, text): WHERE fields LIKE '%{}%' ORDER BY importance DESC LIMIT 10 - """.format(text) + """.format( + text + ) rows = c.execute(query).fetchall() to_return = [] for (importance, chrOffset, fields) in rows: - field_parts = fields.split('\t') - to_return += [{ - 'chr': field_parts[0], - 'txStart': int(field_parts[1]), - 'txEnd': int(field_parts[2]), - 'score': importance, - 'geneName': field_parts[3]}] + field_parts = fields.split("\t") + to_return += [ + { + "chr": field_parts[0], + "txStart": int(field_parts[1]), + "txEnd": int(field_parts[2]), + "score": importance, + "geneName": field_parts[3], + } + ] c.execute(query) - - return to_return diff --git a/tilesets/tests.py b/tilesets/tests.py index a895f6cd..96044570 100644 --- a/tilesets/tests.py +++ b/tilesets/tests.py @@ -25,8 +25,9 @@ logger = logging.getLogger(__name__) -def media_file(filename, sub_dir='uploads'): - ''' + +def media_file(filename, sub_dir="uploads"): + """ Returns path to file in the media uploads directory Parameters: ----------- @@ -36,13 +37,14 @@ def media_file(filename, sub_dir='uploads'): ------- filename: string The path of the file in its place in the media uploads directory - ''' + """ target_dir = op.join(hss.MEDIA_ROOT, sub_dir) target_file = op.join(target_dir, op.basename(filename)) return target_file + def media_file_exists(filename): - ''' + """ Test if a file exists in the media uploads directory Parameters: ----------- @@ -51,11 +53,12 @@ def media_file_exists(filename): Returns: ------- The return value. True for success, False otherwise. - ''' + """ return False if not op.exists(media_file(filename)) else True -def add_file(filename, sub_dir='uploads/data'): - ''' + +def add_file(filename, sub_dir="uploads/data"): + """ Add a file to the media directory and return its path. If a file with the same name exists, don't override it. @@ -67,7 +70,7 @@ def add_file(filename, sub_dir='uploads/data'): ------- filename: string The path of the file in the media directory - ''' + """ target_dir = op.join(hss.MEDIA_ROOT, sub_dir) @@ -78,136 +81,127 @@ def add_file(filename, sub_dir='uploads/data'): if not op.exists(target_file): import shutil - shutil.copyfile(filename, - target_file) - django_file = op.join('uploads', filename) + shutil.copyfile(filename, target_file) + + django_file = op.join("uploads", filename) return django_file def test_list_tilesets(self): - user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) - self.client.login(username='user1', password='pass') + user1 = dcam.User.objects.create_user(username="user1", password="pass") + self.client.login(username="user1", password="pass") # create a project ret = self.client.post( - '/api/v1/projects/', - '{"name": "test project", "owner": "user1", "private": "true" }', - content_type='application/json' - ) - assert(ret.status_code==201) + "/api/v1/projects/", + '{"name": "test project", "owner": "user1", "private": "true" }', + content_type="application/json", + ) + assert ret.status_code == 201 private_project = json.loads(ret.content) # create a project ret = self.client.post( - '/api/v1/projects/', - '{"name": "public project", "owner": "user1", "private": "false" }', - content_type='application/json' - ) - assert(ret.status_code==201) + "/api/v1/projects/", + '{"name": "public project", "owner": "user1", "private": "false" }', + content_type="application/json", + ) + assert ret.status_code == 201 public_project = json.loads(ret.content) - print('public_project', public_project) + print("public_project", public_project) ret = self.client.post( - '/api/v1/tilesets/', + "/api/v1/tilesets/", { - 'datafile': 'one_file', - 'filetype': 'a', - 'datatype': '2', - 'private': 'True', - 'coordSystem': 'hg19', - 'uid': 'bb', - 'project': private_project['uuid'] - - } + "datafile": "one_file", + "filetype": "a", + "datatype": "2", + "private": "True", + "coordSystem": "hg19", + "uid": "bb", + "project": private_project["uuid"], + }, ) - assert(ret.status_code==201) + assert ret.status_code == 201 # create another tileset which isn't associated with this # project ret = self.client.post( - '/api/v1/tilesets/', + "/api/v1/tilesets/", { - 'datafile': 'two_file', - 'filetype': 'a', - 'datatype': '2', - 'private': 'True', - 'coordSystem': 'hg19', - 'uid': 'aa', - 'project': public_project['uuid'], - } + "datafile": "two_file", + "filetype": "a", + "datatype": "2", + "private": "True", + "coordSystem": "hg19", + "uid": "aa", + "project": public_project["uuid"], + }, ) - assert(ret.status_code==201) + assert ret.status_code == 201 ret = self.client.get( - '/api/v1/list_tilesets/?ui={}'.format(private_project['uuid'])) + "/api/v1/list_tilesets/?ui={}".format(private_project["uuid"]) + ) val = json.loads(ret.content) - print("ret.content:",ret.content) + print("ret.content:", ret.content) # it should just return the one dataset - assert(val['count'] == 1) + assert val["count"] == 1 # should return all datasets - ret = self.client.get( - '/api/v1/list_tilesets/?') + ret = self.client.get("/api/v1/list_tilesets/?") val = json.loads(ret.content) print("ret:", ret.content) - assert(val['count'] == 2) + assert val["count"] == 2 # list all the projects - ret = self.client.get( - '/api/v1/list_tilesets/?ac=test') + ret = self.client.get("/api/v1/list_tilesets/?ac=test") val = json.loads(ret.content) # should only return the tileset in test project print("ret:", ret.content) - assert(val['count'] == 1) + assert val["count"] == 1 # list all the projects - ret = self.client.get( - '/api/v1/list_tilesets/?ac=user1') + ret = self.client.get("/api/v1/list_tilesets/?ac=user1") val = json.loads(ret.content) # should only return the tileset in test project print("ret:", ret.content) - assert(val['count'] == 2) + assert val["count"] == 2 self.client.logout() - ret = self.client.get( - '/api/v1/list_tilesets/?') + ret = self.client.get("/api/v1/list_tilesets/?") val = json.loads(ret.content) # the tileset in the private project should no longer be visible print("ret:", ret.content) - assert(val['count'] == 1) + assert val["count"] == 1 def test_create(self): - user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) - self.client.login(username='user1', password='pass') + user1 = dcam.User.objects.create_user(username="user1", password="pass") + self.client.login(username="user1", password="pass") ret = self.client.post( - '/api/v1/projects/', - '{"name": "test project", "owner": "user1" }', - content_type='application/json' - ) + "/api/v1/projects/", + '{"name": "test project", "owner": "user1" }', + content_type="application/json", + ) print("ret:", ret.content) - assert(ret.status_code == 201) + assert ret.status_code == 201 - ts1 = tm.Tileset.objects.create( - name='blah') + ts1 = tm.Tileset.objects.create(name="blah") project = json.loads(ret.content) - print("project", project['uuid']) + print("project", project["uuid"]) - ''' + """ ret = self.client.patch( '/api/v1/projects/{}/'.format(project["uuid"]), content_type='application/json' @@ -218,382 +212,365 @@ def test_create(self): print("ts1:", ts1.uuid) ret = self.client.patch( '/api/v1/projects/{}/'.format(project["uuid"]), - ''' + """ class IngestTests(dt.TestCase): - def test_ingest_with_project(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) - upload_file = open('data/chromSizes.tsv', 'rb') + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") + upload_file = open("data/chromSizes.tsv", "rb") mv = tm.Tileset.objects.create( - datafile=dcfu.SimpleUploadedFile( - upload_file.name, upload_file.read() - ), - filetype='chromsizes-tsv', - datatype='chromsizes', + datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), + filetype="chromsizes-tsv", + datatype="chromsizes", owner=self.user1, coordSystem="hg19_1", ) - dcm.call_command('ingest_tileset', - filename = 'data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig', - filetype='bigwig', datatype='vector', - project_name="some_file") + dcm.call_command( + "ingest_tileset", + filename="data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig", + filetype="bigwig", + datatype="vector", + project_name="some_file", + ) def test_ingest_bigwig(self): with self.assertRaises(dcmb.CommandError): - dcm.call_command('ingest_tileset', - filename = 'data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig', - filetype='bigwig', datatype='vector') + dcm.call_command( + "ingest_tileset", + filename="data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig", + filetype="bigwig", + datatype="vector", + ) - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) - upload_file = open('data/chromSizes.tsv', 'rb') + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") + upload_file = open("data/chromSizes.tsv", "rb") mv = tm.Tileset.objects.create( - datafile=dcfu.SimpleUploadedFile( - upload_file.name, upload_file.read() - ), - filetype='chromsizes-tsv', - datatype='chromsizes', + datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), + filetype="chromsizes-tsv", + datatype="chromsizes", owner=self.user1, coordSystem="hg19_1", ) # this should succeed when the others fail - dcm.call_command('ingest_tileset', - filename = 'data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig', - filetype='bigwig', datatype='vector') + dcm.call_command( + "ingest_tileset", + filename="data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig", + filetype="bigwig", + datatype="vector", + ) with self.assertRaises(dcmb.CommandError): - dcm.call_command('ingest_tileset', - filename = 'data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig', - filetype='bigwig', datatype='vector', coordSystem='a') + dcm.call_command( + "ingest_tileset", + filename="data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig", + filetype="bigwig", + datatype="vector", + coordSystem="a", + ) - upload_file = open('data/chromSizes.tsv', 'rb') + upload_file = open("data/chromSizes.tsv", "rb") mv = tm.Tileset.objects.create( - datafile=dcfu.SimpleUploadedFile( - upload_file.name, upload_file.read() - ), - filetype='chromsizes-tsv', - datatype='chromsizes', + datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), + filetype="chromsizes-tsv", + datatype="chromsizes", owner=self.user1, coordSystem="hg19_2", ) with self.assertRaises(dcmb.CommandError): - dcm.call_command('ingest_tileset', - filename = 'data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig', - filetype='bigwig', datatype='vector') + dcm.call_command( + "ingest_tileset", + filename="data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig", + filetype="bigwig", + datatype="vector", + ) # dcm.call_command('ingest_tileset', filename = 'data/chromSizes.tsv', filetype='chromsizes-tsv', datatype='chromsizes') - #dcm.call_command('ingest_tileset', filename = 'data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig', filetype='bigwig', datatype='vector') + # dcm.call_command('ingest_tileset', filename = 'data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig', filetype='bigwig', datatype='vector') def test_ingest_reordered_bigwig(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) - upload_file = open('data/chromSizes_hg19_reordered.tsv', 'rb') + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") + upload_file = open("data/chromSizes_hg19_reordered.tsv", "rb") mv = tm.Tileset.objects.create( - datafile=dcfu.SimpleUploadedFile( - upload_file.name, upload_file.read() - ), - filetype='chromsizes-tsv', - datatype='chromsizes', + datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), + filetype="chromsizes-tsv", + datatype="chromsizes", owner=self.user1, coordSystem="hg19_r", ) - dcm.call_command('ingest_tileset', - filename = 'data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig', - filetype='bigwig', datatype='vector', - uid='a', - coordSystem='hg19_r') + dcm.call_command( + "ingest_tileset", + filename="data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig", + filetype="bigwig", + datatype="vector", + uid="a", + coordSystem="hg19_r", + ) + + ret = self.client.get("/api/v1/tileset_info/?d=a") + tileset_info = json.loads(ret.content.decode("utf-8")) + assert tileset_info["a"]["chromsizes"][0][0] == "chrM" - ret = self.client.get('/api/v1/tileset_info/?d=a') - tileset_info = json.loads(ret.content.decode('utf-8')) - assert(tileset_info['a']['chromsizes'][0][0] == 'chrM') + ret = self.client.get("/api/v1/tiles/?d=a.22.0") + tile = json.loads(ret.content.decode("utf-8"))["a.22.0"] - ret = self.client.get('/api/v1/tiles/?d=a.22.0') - tile = json.loads(ret.content.decode('utf-8'))['a.22.0'] + ret = self.client.get("/api/v1/tiles/?d=a.22.17") + tile = json.loads(ret.content.decode("utf-8"))["a.22.17"] + assert tile["min_value"] == "NaN" - ret = self.client.get('/api/v1/tiles/?d=a.22.17') - tile = json.loads(ret.content.decode('utf-8'))['a.22.17'] - assert(tile['min_value'] == 'NaN') + ret = self.client.get("/api/v1/tiles/?d=a.22.117") + tile = json.loads(ret.content.decode("utf-8"))["a.22.117"] + assert tile["min_value"] == "NaN" - ret = self.client.get('/api/v1/tiles/?d=a.22.117') - tile = json.loads(ret.content.decode('utf-8'))['a.22.117'] - assert(tile['min_value'] == 'NaN') + # print("tile:", tile) - #print("tile:", tile) - def test_ingest_bigbed(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) - upload_file = open('data/chromSizes_hg38_bbtest.tsv', 'rb') + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") + upload_file = open("data/chromSizes_hg38_bbtest.tsv", "rb") mv = tm.Tileset.objects.create( - datafile=dcfu.SimpleUploadedFile( - upload_file.name, upload_file.read() - ), - filetype='chromsizes-tsv', - datatype='chromsizes', + datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), + filetype="chromsizes-tsv", + datatype="chromsizes", owner=self.user1, coordSystem="hg38_bb", ) - - dcm.call_command('ingest_tileset', - filename = 'data/masterlist_DHSs_733samples_WM20180608_all_mean_signal_colorsMax.bed.bb', - filetype='bigbed', datatype='gene-bed12-annotation', - uid='a', - coordSystem='hg38_bb') - - ret = self.client.get('/api/v1/tileset_info/?d=a') - tileset_info = json.loads(ret.content.decode('utf-8')) - assert(tileset_info['a']['chromsizes'][0][0] == 'chr1') - assert(len(tileset_info['a']['chromsizes']) == 24) - assert(tileset_info['a']['chromsizes'][23][0] == 'chrY') + + dcm.call_command( + "ingest_tileset", + filename="data/masterlist_DHSs_733samples_WM20180608_all_mean_signal_colorsMax.bed.bb", + filetype="bigbed", + datatype="gene-bed12-annotation", + uid="a", + coordSystem="hg38_bb", + ) + + ret = self.client.get("/api/v1/tileset_info/?d=a") + tileset_info = json.loads(ret.content.decode("utf-8")) + assert tileset_info["a"]["chromsizes"][0][0] == "chr1" + assert len(tileset_info["a"]["chromsizes"]) == 24 + assert tileset_info["a"]["chromsizes"][23][0] == "chrY" class TileTests(dt.TestCase): def test_partitioning(self): result = tgt.partition_by_adjacent_tiles(["a.5.0.0", "a.5.0.10"]) - assert(len(result) == 2) + assert len(result) == 2 result = tgt.partition_by_adjacent_tiles(["a.5.0.0", "a.5.0.10", "a.5.0.11"]) - assert(len(result) == 2) + assert len(result) == 2 - result = tgt.partition_by_adjacent_tiles(["a.5.0.0", "a.5.0.10", "a.5.0.11", "a.7.11"]) + result = tgt.partition_by_adjacent_tiles( + ["a.5.0.0", "a.5.0.10", "a.5.0.11", "a.7.11"] + ) - assert(len(result) == 3) + assert len(result) == 3 result = tgt.partition_by_adjacent_tiles(["a.5.0", "a.5.1", "a.5.2"]) - assert(len(result) == 1) + assert len(result) == 1 class BamTests(dt.TestCase): def test_get_tile(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) - bam_file = open('data/SRR1770413.mismatched_bai.bam', 'rb') - bai_file = open('data/SRR1770413.different_index_filename.bai', 'rb') + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") + bam_file = open("data/SRR1770413.mismatched_bai.bam", "rb") + bai_file = open("data/SRR1770413.different_index_filename.bai", "rb") mv = tm.Tileset.objects.create( - datafile=dcfu.SimpleUploadedFile( - bam_file.name, bam_file.read() - ), - indexfile=dcfu.SimpleUploadedFile( - bai_file.name, bai_file.read() - ), - filetype='bam', - datatype='reads', + datafile=dcfu.SimpleUploadedFile(bam_file.name, bam_file.read()), + indexfile=dcfu.SimpleUploadedFile(bai_file.name, bai_file.read()), + filetype="bam", + datatype="reads", coordSystem="unknown", owner=self.user1, - uuid='a' + uuid="a", ) - ret = self.client.get('/api/v1/tileset_info/?d=a') - content = json.loads(ret.content.decode('utf-8')) - assert content['a']['max_width'] > 8e6 + ret = self.client.get("/api/v1/tileset_info/?d=a") + content = json.loads(ret.content.decode("utf-8")) + assert content["a"]["max_width"] > 8e6 - ret = self.client.get('/api/v1/tiles/?d=a.9.0') - content = json.loads(ret.content.decode('utf-8')) + ret = self.client.get("/api/v1/tiles/?d=a.9.0") + content = json.loads(ret.content.decode("utf-8")) # data is stored in a columnar fashion under the heading names - assert len(content['a.9.0']['id']) > 10 + assert len(content["a.9.0"]["id"]) > 10 - ret = self.client.get('/api/v1/tiles/?d=a.0.0') - content = json.loads(ret.content.decode('utf-8')) + ret = self.client.get("/api/v1/tiles/?d=a.0.0") + content = json.loads(ret.content.decode("utf-8")) - assert 'error' in content['a.0.0'] + assert "error" in content["a.0.0"] def test_register_bam_url(self): - ''' + """ Registering a url allows the file to remain on a remote server and be accessed through the local higlass-server - ''' + """ c = dt.Client() - c.login(username='user1', password='pass') + c.login(username="user1", password="pass") uid = slugid.nice() # url = "https://s3.amazonaws.com/pkerp/public/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig" response = c.post( - '/api/v1/register_url/', - json.dumps({ - "fileurl": 'https://s3.amazonaws.com/pkerp/public/SRR1770413.sorted.short.bam', - "indexurl": 'https://s3.amazonaws.com/pkerp/public/SRR1770413.sorted.short.bam.bai', - "uid": uid, - "name": uid, - "datatype": "reads", - "filetype": "bam", - "coordSystem": "hg19" - }), + "/api/v1/register_url/", + json.dumps( + { + "fileurl": "https://s3.amazonaws.com/pkerp/public/SRR1770413.sorted.short.bam", + "indexurl": "https://s3.amazonaws.com/pkerp/public/SRR1770413.sorted.short.bam.bai", + "uid": uid, + "name": uid, + "datatype": "reads", + "filetype": "bam", + "coordSystem": "hg19", + } + ), content_type="application/json", ) self.assertEqual(200, response.status_code, response.content) - response = c.get('/api/v1/tilesets/') - record_matches = [result for result in response.json()['results'] if result['uuid'] == uid] + response = c.get("/api/v1/tilesets/") + record_matches = [ + result for result in response.json()["results"] if result["uuid"] == uid + ] assert len(record_matches) == 1 - assert record_matches[0]['name'] == uid + assert record_matches[0]["name"] == uid obj = tm.Tileset.objects.get(uuid=uid) assert "SRR1770413" in obj.datafile.path assert obj.temporary == True - response = c.get(f'/api/v1/tileset_info/?d={uid}') + response = c.get(f"/api/v1/tileset_info/?d={uid}") assert response.status_code == 200 - response = c.get('/api/v1/tiles/?d={uuid}.0.0'.format(uuid=uid)) + response = c.get("/api/v1/tiles/?d={uuid}.0.0".format(uuid=uid)) assert response.status_code == 200 - response = c.get('/api/v1/tiles/?d={uuid}.0.0'.format(uuid=uid)) + response = c.get("/api/v1/tiles/?d={uuid}.0.0".format(uuid=uid)) assert response.status_code == 200 cont = json.loads(response.content) - assert 'error' in cont[f'{uid}.0.0'] + assert "error" in cont[f"{uid}.0.0"] - response = c.get('/api/v1/tiles/?d={uuid}.9.0'.format(uuid=uid)) + response = c.get("/api/v1/tiles/?d={uuid}.9.0".format(uuid=uid)) assert response.status_code == 200 cont = json.loads(response.content) - assert len(cont[f'{uid}.9.0']['id']) > 10 + assert len(cont[f"{uid}.9.0"]["id"]) > 10 class MultivecTests(dt.TestCase): def test_get_tile(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) - upload_file = open('data/sample.bed.multires.mv5', 'rb') + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") + upload_file = open("data/sample.bed.multires.mv5", "rb") mv = tm.Tileset.objects.create( - datafile=dcfu.SimpleUploadedFile( - upload_file.name, upload_file.read() - ), - filetype='multivec', - datatype='multivec', + datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), + filetype="multivec", + datatype="multivec", coordSystem="hg38", owner=self.user1, - uuid='a' + uuid="a", ) - ''' + """ ret = self.client.get('/api/v1/tiles/?d=a.12.0') content = json.loads(ret.content) r = base64.decodestring(content['a.12.0']['dense'].encode('utf-8')) q = np.frombuffer(r, dtype=np.float16) - ''' + """ - ret = self.client.get('/api/v1/tiles/?d=a.11.0') - content = json.loads(ret.content.decode('utf-8')) - r = base64.decodestring(content['a.11.0']['dense'].encode('utf-8')) + ret = self.client.get("/api/v1/tiles/?d=a.11.0") + content = json.loads(ret.content.decode("utf-8")) + r = base64.decodestring(content["a.11.0"]["dense"].encode("utf-8")) q = np.frombuffer(r, dtype=np.float16) class ChromosomeSizes(dt.TestCase): def test_list_chromsizes(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) - upload_file = open('data/chromSizes.tsv', 'rb') + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") + upload_file = open("data/chromSizes.tsv", "rb") self.chroms = tm.Tileset.objects.create( - datafile=dcfu.SimpleUploadedFile( - upload_file.name, upload_file.read() - ), - filetype='chromsizes-tsv', - datatype='chromsizes', + datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), + filetype="chromsizes-tsv", + datatype="chromsizes", coordSystem="hg19", owner=self.user1, - uuid='cs-hg19' + uuid="cs-hg19", ) ret = json.loads( - self.client.get( - '/api/v1/available-chrom-sizes/' - ).content.decode('utf-8') + self.client.get("/api/v1/available-chrom-sizes/").content.decode("utf-8") ) - assert(ret["count"] == 1) - assert(len(ret["results"]) == 1) + assert ret["count"] == 1 + assert len(ret["results"]) == 1 - ret = self.client.get('/api/v1/chrom-sizes/?id=cs-hg19') + ret = self.client.get("/api/v1/chrom-sizes/?id=cs-hg19") - assert(ret.status_code == 200) - assert(len(ret.content) > 300) + assert ret.status_code == 200 + assert len(ret.content) > 300 - ret = self.client.get('/api/v1/chrom-sizes/?id=cs-hg19&type=json') + ret = self.client.get("/api/v1/chrom-sizes/?id=cs-hg19&type=json") - data = json.loads(ret.content.decode('utf-8')) - assert(ret.status_code == 200) - assert('chr1' in data) + data = json.loads(ret.content.decode("utf-8")) + assert ret.status_code == 200 + assert "chr1" in data - ret = self.client.get( - '/api/v1/chrom-sizes/?id=cs-hg19&type=json&cum=1' - ) + ret = self.client.get("/api/v1/chrom-sizes/?id=cs-hg19&type=json&cum=1") - data = json.loads(ret.content.decode('utf-8')) - assert(ret.status_code == 200) - assert('offset' in data['chr1']) + data = json.loads(ret.content.decode("utf-8")) + assert ret.status_code == 200 + assert "offset" in data["chr1"] def test_chromsizes_from_cooler(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") upload_file = open( - 'data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool', - 'rb' + "data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool", "rb" ) self.chroms = tm.Tileset.objects.create( - datafile=dcfu.SimpleUploadedFile( - upload_file.name, upload_file.read() - ), - filetype='cooler', - datatype='matrix', - coordSystem='hg19', + datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), + filetype="cooler", + datatype="matrix", + coordSystem="hg19", owner=self.user1, - uuid='cooler-dixon' + uuid="cooler-dixon", ) - ret = self.client.get('/api/v1/chrom-sizes/?id=cooler-dixon') + ret = self.client.get("/api/v1/chrom-sizes/?id=cooler-dixon") - assert(ret.status_code == 200) - assert(len(ret.content) > 300) + assert ret.status_code == 200 + assert len(ret.content) > 300 - ret = self.client.get('/api/v1/chrom-sizes/?id=cooler-dixon&type=json') + ret = self.client.get("/api/v1/chrom-sizes/?id=cooler-dixon&type=json") - data = json.loads(ret.content.decode('utf-8')) - assert(ret.status_code == 200) - assert('chr1' in data) + data = json.loads(ret.content.decode("utf-8")) + assert ret.status_code == 200 + assert "chr1" in data - ret = self.client.get( - '/api/v1/chrom-sizes/?id=cooler-dixon&type=json&cum=1' - ) + ret = self.client.get("/api/v1/chrom-sizes/?id=cooler-dixon&type=json&cum=1") - data = json.loads(ret.content.decode('utf-8')) - assert(ret.status_code == 200) - assert('offset' in data['chr1']) + data = json.loads(ret.content.decode("utf-8")) + assert ret.status_code == 200 + assert "offset" in data["chr1"] class TilesetModelTest(dt.TestCase): def test_to_string(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") + upload_file = open( + "data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool", "rb" ) - upload_file = open('data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool', 'rb') self.cooler = tm.Tileset.objects.create( - datafile=dcfu.SimpleUploadedFile(upload_file.name, - upload_file.read()), - filetype='cooler', + datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), + filetype="cooler", owner=self.user1, - uuid='x1x' + uuid="x1x", ) cooler_string = str(self.cooler) @@ -602,23 +579,23 @@ def test_to_string(self): class UnknownTilesetTypeTest(dt.TestCase): def setUp(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") - upload_file = open('data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool', 'rb') + upload_file = open( + "data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool", "rb" + ) self.cooler = tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), - filetype='bar', - datatype='foo', + filetype="bar", + datatype="foo", owner=self.user1, - uuid='cli-huge-test' + uuid="cli-huge-test", ) def test_file_size(self): # make sure that the returned tiles are not overly large - ret = self.client.get('/api/v1/tiles/?d=cli-huge-test.0.0.0') - val = json.loads(ret.content.decode('utf-8')) + ret = self.client.get("/api/v1/tiles/?d=cli-huge-test.0.0.0") + val = json.loads(ret.content.decode("utf-8")) # 32 bit: 349528 # 16 bit: 174764 @@ -626,22 +603,22 @@ def test_file_size(self): class TilesizeTest(dt.TestCase): def setUp(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") - upload_file = open('data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool', 'rb') + upload_file = open( + "data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool", "rb" + ) self.cooler = tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), - filetype='cooler', + filetype="cooler", owner=self.user1, - uuid='x1x' + uuid="x1x", ) def test_file_size(self): # make sure that the returned tiles are not overly large - ret = self.client.get('/api/v1/tiles/?d=x1x.0.0.0') - val = json.loads(ret.content.decode('utf-8')) + ret = self.client.get("/api/v1/tiles/?d=x1x.0.0.0") + val = json.loads(ret.content.decode("utf-8")) # 32 bit: 349528 # 16 bit: 174764 @@ -649,264 +626,244 @@ def test_file_size(self): class ViewConfTest(dt.TestCase): def setUp(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") - upload_json_text = json.dumps({'hi': 'there'}) + upload_json_text = json.dumps({"hi": "there"}) - self.viewconf = tm.ViewConf.objects.create( - viewconf=upload_json_text, uuid='md') + self.viewconf = tm.ViewConf.objects.create(viewconf=upload_json_text, uuid="md") def test_viewconf(self): - ret = self.client.get('/api/v1/viewconfs/?d=md') + ret = self.client.get("/api/v1/viewconfs/?d=md") - contents = json.loads(ret.content.decode('utf-8')) - assert('hi' in contents) + contents = json.loads(ret.content.decode("utf-8")) + assert "hi" in contents def test_viewconfs(self): ret = self.client.post( - '/api/v1/viewconfs/', + "/api/v1/viewconfs/", '{"uid": "123", "viewconf":{"hello": "sir"}}', - content_type="application/json" + content_type="application/json", ) - contents = json.loads(ret.content.decode('utf-8')) + contents = json.loads(ret.content.decode("utf-8")) if hss.UPLOAD_ENABLED: - self.assertIn('uid', contents) - self.assertEqual(contents['uid'], '123') + self.assertIn("uid", contents) + self.assertEqual(contents["uid"], "123") - url = '/api/v1/viewconfs/?d=123' + url = "/api/v1/viewconfs/?d=123" ret = self.client.get(url) - contents = json.loads(ret.content.decode('utf-8')) + contents = json.loads(ret.content.decode("utf-8")) - assert('hello' in contents) + assert "hello" in contents else: self.assertEquals(ret.status_code, 403) def test_duplicate_uid_errors(self): ret1 = self.client.post( - '/api/v1/viewconfs/', + "/api/v1/viewconfs/", '{"uid": "dupe", "viewconf":{"try": "first"}}', - content_type="application/json" - ) - self.assertEqual( - ret1.status_code, - 200 if hss.UPLOAD_ENABLED else 403 + content_type="application/json", ) + self.assertEqual(ret1.status_code, 200 if hss.UPLOAD_ENABLED else 403) if hss.UPLOAD_ENABLED: # TODO: This will bubble up as a 500, when bad user input should # really be 4xx. ret = self.client.post( - '/api/v1/viewconfs/', + "/api/v1/viewconfs/", '{"uid": "dupe", "viewconf":{"try": "second"}}', - content_type="application/json" + content_type="application/json", ) - assert(ret.status_code == 400) + assert ret.status_code == 400 class PermissionsTest(dt.TestCase): def setUp(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") - self.user2 = dcam.User.objects.create_user( - username='user2', password='pass' - ) + self.user2 = dcam.User.objects.create_user(username="user2", password="pass") def test_permissions(self): c1 = dt.Client() - f = open('data/tiny.txt', 'rb') + f = open("data/tiny.txt", "rb") test_tileset = { - 'datafile': f, - 'filetype': 'hitile', - 'datatype': 'vector', - 'uid': 'bb', - 'private': 'True', - 'coordSystem': 'hg19', - 'name': "tr1" + "datafile": f, + "filetype": "hitile", + "datatype": "vector", + "uid": "bb", + "private": "True", + "coordSystem": "hg19", + "name": "tr1", } - response = c1.post( - '/api/v1/tilesets/', - test_tileset, - format='multipart' - ) + response = c1.post("/api/v1/tilesets/", test_tileset, format="multipart") # user must be logged in to create objects - assert(response.status_code == 403) + assert response.status_code == 403 f.close() - c1.login(username='user1', password='pass') + c1.login(username="user1", password="pass") - fname = 'data/tiny.txt' - fhandle = open(fname, 'rb') + fname = "data/tiny.txt" + fhandle = open(fname, "rb") test_tileset = { - 'datafile': fhandle, - 'filetype': 'hitile', - 'datatype': 'vector', - 'uid': 'cc', - 'private': 'True', - 'coordSystem': 'hg19', - 'name': "tr2" + "datafile": fhandle, + "filetype": "hitile", + "datatype": "vector", + "uid": "cc", + "private": "True", + "coordSystem": "hg19", + "name": "tr2", } - response = c1.post( - '/api/v1/tilesets/', - test_tileset, - format='multipart' - ) + response = c1.post("/api/v1/tilesets/", test_tileset, format="multipart") fhandle.close() if hss.UPLOAD_ENABLED: # creating datasets is allowed if we're logged in - assert(response.status_code == 201) + assert response.status_code == 201 - ret = json.loads(response.content.decode('utf-8')) + ret = json.loads(response.content.decode("utf-8")) # update media filename for whatever name the server ended up using (i.e., in case of duplicates, a random suffix is added) - assert('datafile' in ret) - fname = op.basename(ret['datafile']) + assert "datafile" in ret + fname = op.basename(ret["datafile"]) # test that said media file exists - assert(media_file_exists(fname)) + assert media_file_exists(fname) c2 = dt.Client() - c2.login(username='user2', password='pass') + c2.login(username="user2", password="pass") # user2 should not be able to delete the tileset created by user1 - resp = c2.delete('/api/v1/tilesets/' + ret['uuid'] + "/") - assert(resp.status_code == 403) + resp = c2.delete("/api/v1/tilesets/" + ret["uuid"] + "/") + assert resp.status_code == 403 # the media file should still exist - assert(media_file_exists(fname)) - + assert media_file_exists(fname) # user2 should not be able to rename the tileset created by user1 - resp = c2.put('/api/v1/tilesets/' + ret['uuid'] + "/", data='{"name":"newname"}', content_type='application/json') - assert(resp.status_code == 403) + resp = c2.put( + "/api/v1/tilesets/" + ret["uuid"] + "/", + data='{"name":"newname"}', + content_type="application/json", + ) + assert resp.status_code == 403 # tileset should still be in the database and on the filesystem resp = c1.get("/api/v1/tilesets/") - assert(json.loads(resp.content.decode('utf-8'))['count'] == 1) - assert(media_file_exists(fname)) - + assert json.loads(resp.content.decode("utf-8"))["count"] == 1 + assert media_file_exists(fname) # user1 should be able to rename or modify their tileset - resp = c1.patch('/api/v1/tilesets/' + ret['uuid'] + "/", data='{"name":"newname"}', content_type='application/json') - assert(resp.status_code == 200) + resp = c1.patch( + "/api/v1/tilesets/" + ret["uuid"] + "/", + data='{"name":"newname"}', + content_type="application/json", + ) + assert resp.status_code == 200 # apply GET on uuid to ensure that tileset has the newly modified name - resp = c1.get("/api/v1/tilesets/" + ret['uuid'] + '/') - assert(json.loads(resp.content.decode('utf-8'))['name'] == 'newname') + resp = c1.get("/api/v1/tilesets/" + ret["uuid"] + "/") + assert json.loads(resp.content.decode("utf-8"))["name"] == "newname" # the media file should still exist with the same name - assert(media_file_exists(fname)) + assert media_file_exists(fname) # user1 should be able to delete his/her own tileset from the database - resp = c1.delete('/api/v1/tilesets/' + ret['uuid'] + "/") + resp = c1.delete("/api/v1/tilesets/" + ret["uuid"] + "/") resp = c1.get("/api/v1/tilesets/") - assert(resp.status_code == 200) - assert(json.loads(resp.content.decode('utf-8'))['count'] == 0) + assert resp.status_code == 200 + assert json.loads(resp.content.decode("utf-8"))["count"] == 0 # the media file should no longer exist, as well - assert(not media_file_exists(fname)) - + assert not media_file_exists(fname) c3 = dt.Client() - resp = c3.get('/api/v1/tilesets/') + resp = c3.get("/api/v1/tilesets/") # unauthenticated users should be able to see the (public) tileset # list - assert(resp.status_code == 200) + assert resp.status_code == 200 else: - assert(response.status_code == 403) + assert response.status_code == 403 def test_filter(self): c1 = dt.Client() - c1.login(username='user1', password='pass') - f = open('data/tiny.txt', 'rb') + c1.login(username="user1", password="pass") + f = open("data/tiny.txt", "rb") test_tileset = { - 'datafile': f, - 'filetype': 'hitile', - 'datatype': 'vector', - 'uid': 'bb', - 'private': 'True', - 'coordSystem': 'hg19', - 'name': "tr1" + "datafile": f, + "filetype": "hitile", + "datatype": "vector", + "uid": "bb", + "private": "True", + "coordSystem": "hg19", + "name": "tr1", } - response = c1.post( - '/api/v1/tilesets/', - test_tileset, - format='multipart' - ) + response = c1.post("/api/v1/tilesets/", test_tileset, format="multipart") f.close() - f = open('data/tiny.txt', 'rb') + f = open("data/tiny.txt", "rb") test_tileset = { - 'datafile': f, - 'filetype': 'hitile', - 'datatype': 'vector', - 'uid': 'cc', - 'private': 'True', - 'coordSystem': 'hg19', - 'name': "tr2" + "datafile": f, + "filetype": "hitile", + "datatype": "vector", + "uid": "cc", + "private": "True", + "coordSystem": "hg19", + "name": "tr2", } - response = c1.post( - '/api/v1/tilesets/', - test_tileset, - format='multipart' - ) + response = c1.post("/api/v1/tilesets/", test_tileset, format="multipart") f.close() - assert(response is not None) + assert response is not None - ret = json.loads(c1.get('/api/v1/tilesets/').content.decode('utf-8')) - assert(ret['count'] == 2) + ret = json.loads(c1.get("/api/v1/tilesets/").content.decode("utf-8")) + assert ret["count"] == 2 - ret = json.loads(c1.get('/api/v1/tilesets/?ac=tr2').content.decode('utf-8')) - assert(ret['count'] == 1) + ret = json.loads(c1.get("/api/v1/tilesets/?ac=tr2").content.decode("utf-8")) + assert ret["count"] == 1 class BigWigTest(dt.TestCase): def setUp(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") - upload_file = open('data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig', 'rb') - #x = upload_file.read() + upload_file = open( + "data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig", "rb" + ) + # x = upload_file.read() self.tileset = tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), - filetype='bigwig', - datatype='vector', + filetype="bigwig", + datatype="vector", owner=self.user1, - coordSystem='hg19', - coordSystem2='hg19', + coordSystem="hg19", + coordSystem2="hg19", name="x", - uuid='bw') + uuid="bw", + ) def test_get_tileset_info(self): c1 = dt.Client() - ret = json.loads(c1.get('/api/v1/tileset_info/?d=bw').content.decode('utf-8')) + ret = json.loads(c1.get("/api/v1/tileset_info/?d=bw").content.decode("utf-8")) def test_get_tiles(self): - ''' + """ Try to retrieve some tiles from this file - ''' + """ c1 = dt.Client() - c1.login(username='user1', password='pass') + c1.login(username="user1", password="pass") # make sure that the dataset has been added - ret = json.loads(c1.get('/api/v1/tilesets/?d=bw').content.decode('utf-8')) - assert(ret['count'] == 1) + ret = json.loads(c1.get("/api/v1/tilesets/?d=bw").content.decode("utf-8")) + assert ret["count"] == 1 # try to retrieve the top level tile # ret = json.loads(c1.get('/api/v1/tiles/?d=bw.0.0').content.decode('utf-8')) @@ -914,46 +871,52 @@ def test_get_tiles(self): # retrieve a tile that lies completely beyond the end of # the assembly - ret = json.loads(c1.get('/api/v1/tiles/?d=bw.22.4194303').content.decode('utf-8')) + ret = json.loads( + c1.get("/api/v1/tiles/?d=bw.22.4194303").content.decode("utf-8") + ) class BigBedTest(dt.TestCase): def setUp(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") - upload_file = open('data/masterlist_DHSs_733samples_WM20180608_all_mean_signal_colorsMax.bed.bb', 'rb') + upload_file = open( + "data/masterlist_DHSs_733samples_WM20180608_all_mean_signal_colorsMax.bed.bb", + "rb", + ) self.tileset = tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), - filetype='bigbed', - datatype='gene-bed12-annotation', + filetype="bigbed", + datatype="gene-bed12-annotation", owner=self.user1, - coordSystem='hg38_bb', - coordSystem2='', + coordSystem="hg38_bb", + coordSystem2="", name="masterlist_DHSs_733samples_WM20180608_all_mean_signal_colorsMax.bed.bb", - uuid='bb') + uuid="bb", + ) def test_get_tileset_info(self): c1 = dt.Client() try: - ret = json.loads(c1.get('/api/v1/tileset_info/?d=bb').content.decode('utf-8')) - assert(len(ret['bb']['chromsizes']) == 24) + ret = json.loads( + c1.get("/api/v1/tileset_info/?d=bb").content.decode("utf-8") + ) + assert len(ret["bb"]["chromsizes"]) == 24 except OSError: pass def test_get_tiles(self): - ''' + """ Try to retrieve some tiles from this file - ''' + """ c1 = dt.Client() - c1.login(username='user1', password='pass') + c1.login(username="user1", password="pass") # make sure that the dataset has been added try: - ret = json.loads(c1.get('/api/v1/tilesets/?d=bb').content.decode('utf-8')) - assert(ret['count'] == 1) + ret = json.loads(c1.get("/api/v1/tilesets/?d=bb").content.decode("utf-8")) + assert ret["count"] == 1 except OSError: pass @@ -963,141 +926,147 @@ def test_get_tiles(self): # try to retrieve a tile from within chr1 try: - ret = json.loads(c1.get('/api/v1/tiles/?d=bb.14.13').content.decode('utf-8')) - assert(len(ret) == 1) + ret = json.loads( + c1.get("/api/v1/tiles/?d=bb.14.13").content.decode("utf-8") + ) + assert len(ret) == 1 except OSError: pass class CoolerTest(dt.TestCase): def setUp(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") - upload_file = open('data/Dixon2012-J1-NcoI-R1-filtered.100kb.multires.cool', - 'rb') + upload_file = open( + "data/Dixon2012-J1-NcoI-R1-filtered.100kb.multires.cool", "rb" + ) self.tileset = tm.Tileset.objects.create( - datafile=dcfu.SimpleUploadedFile(upload_file.name, - upload_file.read()), - filetype='cooler', - datatype='matrix', - owner=self.user1, - coordSystem='hg19', - coordSystem2='hg19', - name="x", - uuid='md') + datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), + filetype="cooler", + datatype="matrix", + owner=self.user1, + coordSystem="hg19", + coordSystem2="hg19", + name="x", + uuid="md", + ) self.tileset = tm.Tileset.objects.create( - datafile=dcfu.SimpleUploadedFile(upload_file.name, - upload_file.read()), - filetype='cooler', - datatype='matrix', + datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), + filetype="cooler", + datatype="matrix", owner=self.user1, - coordSystem='hg19', - coordSystem2='hg19', + coordSystem="hg19", + coordSystem2="hg19", name="t", - uuid='rd') + uuid="rd", + ) self.tileset = tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), - filetype='cooler', - datatype='matrix', + filetype="cooler", + datatype="matrix", owner=self.user1, - coordSystem='hg19', - coordSystem2='hg19', + coordSystem="hg19", + coordSystem2="hg19", name="Z", - uuid='a') + uuid="a", + ) self.tileset = tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), - filetype='cooler', - datatype='matrix', + filetype="cooler", + datatype="matrix", owner=self.user1, - coordSystem='hg19', - coordSystem2='hg19', + coordSystem="hg19", + coordSystem2="hg19", name="v", - uuid='sd1') + uuid="sd1", + ) - upload_file = open('data/hic-resolutions.cool', 'rb') + upload_file = open("data/hic-resolutions.cool", "rb") self.tileset = tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), - filetype='cooler', - datatype='matrix', + filetype="cooler", + datatype="matrix", owner=self.user1, - coordSystem='x', - coordSystem2='x', + coordSystem="x", + coordSystem2="x", name="nuhr", - uuid='nuhr') + uuid="nuhr", + ) def test_order_by(self): - ''' + """ Test to make sure that tilesets are correctly ordered when returned - ''' - ret = self.client.get('/api/v1/tilesets/?o=uuid') - contents = json.loads(ret.content.decode('utf-8')) - - uuids = [r['uuid'] for r in contents['results']] - assert(uuids[0] < uuids[1]) + """ + ret = self.client.get("/api/v1/tilesets/?o=uuid") + contents = json.loads(ret.content.decode("utf-8")) - ret = self.client.get('/api/v1/tilesets/?o=uuid&r=1') - contents = json.loads(ret.content.decode('utf-8')) + uuids = [r["uuid"] for r in contents["results"]] + assert uuids[0] < uuids[1] - uuids = [r['uuid'] for r in contents['results']] - assert(uuids[0] > uuids[1]) + ret = self.client.get("/api/v1/tilesets/?o=uuid&r=1") + contents = json.loads(ret.content.decode("utf-8")) - ret = self.client.get('/api/v1/tilesets/?o=name') - contents = json.loads(ret.content.decode('utf-8')) + uuids = [r["uuid"] for r in contents["results"]] + assert uuids[0] > uuids[1] - names = [r['name'] for r in contents['results']] - assert(names[0] < names[1]) + ret = self.client.get("/api/v1/tilesets/?o=name") + contents = json.loads(ret.content.decode("utf-8")) - ret = self.client.get('/api/v1/tilesets/?o=name&r=1') - contents = json.loads(ret.content.decode('utf-8')) + names = [r["name"] for r in contents["results"]] + assert names[0] < names[1] - names = [r['name'] for r in contents['results']] + ret = self.client.get("/api/v1/tilesets/?o=name&r=1") + contents = json.loads(ret.content.decode("utf-8")) - assert(names[0].lower() > names[1].lower()) - assert(names[0].lower() > names[-1].lower()) + names = [r["name"] for r in contents["results"]] + assert names[0].lower() > names[1].lower() + assert names[0].lower() > names[-1].lower() def test_transforms(self): - ''' + """ Try to get different transforms of the same tileset - ''' - ret = self.client.get('/api/v1/tileset_info/?d=md') - contents = json.loads(ret.content.decode('utf-8')) + """ + ret = self.client.get("/api/v1/tileset_info/?d=md") + contents = json.loads(ret.content.decode("utf-8")) - assert('transforms' in contents['md']) - assert(contents['md']['transforms'][0]['name'] == 'ICE') + assert "transforms" in contents["md"] + assert contents["md"]["transforms"][0]["name"] == "ICE" - ret = self.client.get('/api/v1/tiles/?d=md.0.0.0.default') - contents = json.loads(ret.content.decode('utf-8')) + ret = self.client.get("/api/v1/tiles/?d=md.0.0.0.default") + contents = json.loads(ret.content.decode("utf-8")) - ret = self.client.get('/api/v1/tiles/?d=md.0.0.0.none') - contents1 = json.loads(ret.content.decode('utf-8')) + ret = self.client.get("/api/v1/tiles/?d=md.0.0.0.none") + contents1 = json.loads(ret.content.decode("utf-8")) # make sure that different normalization methods result # in different data being returned - assert(contents['md.0.0.0.default']['dense'] != contents1['md.0.0.0.none']['dense']) + assert ( + contents["md.0.0.0.default"]["dense"] != contents1["md.0.0.0.none"]["dense"] + ) def test_unbalanced(self): - ''' + """ Try to get tiles from an unbalanced dataset - ''' - upload_file = open('data/G15509.K-562.2_sampleDown.multires.cool', 'rb') + """ + upload_file = open("data/G15509.K-562.2_sampleDown.multires.cool", "rb") tileset = tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), - filetype='cooler', - datatype='matrix', + filetype="cooler", + datatype="matrix", owner=self.user1, - uuid='g1a') + uuid="g1a", + ) - ret = self.client.get('/api/v1/tiles/?d=g1a.0.0.0') - contents = json.loads(ret.content.decode('utf-8')) + ret = self.client.get("/api/v1/tiles/?d=g1a.0.0.0") + contents = json.loads(ret.content.decode("utf-8")) - self.assertIn('g1a.0.0.0', contents) + self.assertIn("g1a.0.0.0", contents) """ def test_tile_multiresolution_consistency(self): @@ -1141,40 +1110,43 @@ def test_tile_multiresolution_consistency(self): def test_tile_nans(self): tileset = tm.Tileset.objects.create( - datafile=add_file('data/Dixon2012-J1-NcoI-R1-filtered.100kb.multires.cool'), - filetype='cooler', - datatype='matrix', + datafile=add_file("data/Dixon2012-J1-NcoI-R1-filtered.100kb.multires.cool"), + filetype="cooler", + datatype="matrix", owner=self.user1, - uuid='aa') + uuid="aa", + ) - tile_id = 'aa.4.5.5' - ret = self.client.get('/api/v1/tiles/?d={}'.format(tile_id)) - content = json.loads(ret.content.decode('utf-8')); + tile_id = "aa.4.5.5" + ret = self.client.get("/api/v1/tiles/?d={}".format(tile_id)) + content = json.loads(ret.content.decode("utf-8")) data = content[tile_id] def test_tile_symmetry(self): - ''' + """ Make sure that tiles are symmetric - ''' - upload_file = open('data/Dixon2012-J1-NcoI-R1-filtered.100kb.multires.cool', 'rb') + """ + upload_file = open( + "data/Dixon2012-J1-NcoI-R1-filtered.100kb.multires.cool", "rb" + ) tileset = tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), - filetype='cooler', - datatype='matrix', + filetype="cooler", + datatype="matrix", owner=self.user1, - uuid='aa') - - ret = self.client.get('/api/v1/tiles/?d=aa.0.0.0') + uuid="aa", + ) + ret = self.client.get("/api/v1/tiles/?d=aa.0.0.0") - contents = json.loads(ret.content.decode('utf-8')) + contents = json.loads(ret.content.decode("utf-8")) import base64 - r = base64.decodestring(contents['aa.0.0.0']['dense'].encode('utf-8')) - q = np.frombuffer(r, dtype=np.float32) - q = q.reshape((256,256)) + r = base64.decodestring(contents["aa.0.0.0"]["dense"].encode("utf-8")) + q = np.frombuffer(r, dtype=np.float32) + q = q.reshape((256, 256)) """ def test_tile_boundary(self): @@ -1200,170 +1172,169 @@ def test_tile_boundary(self): assert(tile[(5,6)][-1] == 0.) """ - def test_get_tileset_info(self): - ret = self.client.get('/api/v1/tileset_info/?d=md') + ret = self.client.get("/api/v1/tileset_info/?d=md") - contents = json.loads(ret.content.decode('utf-8')) + contents = json.loads(ret.content.decode("utf-8")) - assert('md' in contents) - assert('min_pos' in contents['md']) - assert(contents['md']['coordSystem'] == 'hg19') + assert "md" in contents + assert "min_pos" in contents["md"] + assert contents["md"]["coordSystem"] == "hg19" ### test getting tileset info from files with non-powers of two resolutions - ret = self.client.get('/api/v1/tileset_info/?d=nuhr') + ret = self.client.get("/api/v1/tileset_info/?d=nuhr") - contents = json.loads(ret.content.decode('utf-8')) - assert('nuhr' in contents) + contents = json.loads(ret.content.decode("utf-8")) + assert "nuhr" in contents def test_get_multi_tiles(self): - ret = self.client.get('/api/v1/tiles/?d=md.7.92.97&d=md.7.92.98&d=md.7.93.97&d=md.7.93.98&d=md.7.93.21') - content = json.loads(ret.content.decode('utf-8')) + ret = self.client.get( + "/api/v1/tiles/?d=md.7.92.97&d=md.7.92.98&d=md.7.93.97&d=md.7.93.98&d=md.7.93.21" + ) + content = json.loads(ret.content.decode("utf-8")) - assert('md.7.92.97' in content) - assert('dense' in content['md.7.92.97']) + assert "md.7.92.97" in content + assert "dense" in content["md.7.92.97"] def test_get_tiles(self): # this should fail in some manner because the tile is out of # bounds of the dataset - ret = self.client.get('/api/v1/tiles/?d=nuhr.2.0.0') - content = json.loads(ret.content.decode('utf-8')) + ret = self.client.get("/api/v1/tiles/?d=nuhr.2.0.0") + content = json.loads(ret.content.decode("utf-8")) - assert('nuhr.2.0.0' in content) - assert('dense' in content['nuhr.2.0.0']) + assert "nuhr.2.0.0" in content + assert "dense" in content["nuhr.2.0.0"] return # this is to ensure that no exception is thrown - ret = self.client.get('/api/v1/tiles/?d=nuhr.2.12.13') - content = json.loads(ret.content.decode('utf-8')) + ret = self.client.get("/api/v1/tiles/?d=nuhr.2.12.13") + content = json.loads(ret.content.decode("utf-8")) - assert('nuhr.2.0.0' in content) - assert('dense' in content['nuhr.2.0.0']) + assert "nuhr.2.0.0" in content + assert "dense" in content["nuhr.2.0.0"] return - ret = self.client.get('/api/v1/tiles/?d=md.7.92.97') - content = json.loads(ret.content.decode('utf-8')) + ret = self.client.get("/api/v1/tiles/?d=md.7.92.97") + content = json.loads(ret.content.decode("utf-8")) - assert('md.7.92.97' in content) - assert('dense' in content['md.7.92.97']) + assert "md.7.92.97" in content + assert "dense" in content["md.7.92.97"] def test_get_empty_tiles(self): # this test is here to ensure that the function call doesn't # throw an error because this tile has no data - ret = self.client.get('/api/v1/tiles/?d=md.7.127.127') - content = json.loads(ret.content.decode('utf-8')) + ret = self.client.get("/api/v1/tiles/?d=md.7.127.127") + content = json.loads(ret.content.decode("utf-8")) class SuggestionsTest(dt.TestCase): - ''' + """ Test gene suggestions - ''' + """ + def setUp(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") - upload_file = open('data/gene_annotations.short.db', 'rb') - #x = upload_file.read() + upload_file = open("data/gene_annotations.short.db", "rb") + # x = upload_file.read() self.tileset = tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), - filetype='beddb', - datatype='gene-annotations', + filetype="beddb", + datatype="gene-annotations", owner=self.user1, - uuid='sut', - coordSystem='hg19' - ) + uuid="sut", + coordSystem="hg19", + ) def test_suggest(self): # shouldn't be found and shouldn't raise an error - ret = self.client.get('/api/v1/suggest/?d=xx&ac=r') + ret = self.client.get("/api/v1/suggest/?d=xx&ac=r") - ret = self.client.get('/api/v1/suggest/?d=sut&ac=r') - suggestions = json.loads(ret.content.decode('utf-8')) + ret = self.client.get("/api/v1/suggest/?d=sut&ac=r") + suggestions = json.loads(ret.content.decode("utf-8")) self.assertGreater(len(suggestions), 0) - self.assertGreater(suggestions[0]['score'], suggestions[1]['score']) + self.assertGreater(suggestions[0]["score"], suggestions[1]["score"]) - ret = self.client.get('/api/v1/suggest/?d=sut&ac=r') - suggestions = json.loads(ret.content.decode('utf-8')) + ret = self.client.get("/api/v1/suggest/?d=sut&ac=r") + suggestions = json.loads(ret.content.decode("utf-8")) self.assertGreater(len(suggestions), 0) - self.assertGreater(suggestions[0]['score'], suggestions[1]['score']) + self.assertGreater(suggestions[0]["score"], suggestions[1]["score"]) class FileUploadTest(dt.TestCase): - ''' + """ Test file upload functionality - ''' + """ + def setUp(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") def test_upload_file(self): c = dt.Client() - c.login(username='user1', password='pass') + c.login(username="user1", password="pass") - f = open('data/tiny.txt', 'rb') + f = open("data/tiny.txt", "rb") response = c.post( - '/api/v1/tilesets/', + "/api/v1/tilesets/", { - 'datafile': f, - 'filetype': 'hitile', - 'datatype': 'vector', - 'uid': 'bb', - 'private': 'True', - 'coordSystem': 'hg19' + "datafile": f, + "filetype": "hitile", + "datatype": "vector", + "uid": "bb", + "private": "True", + "coordSystem": "hg19", }, - format='multipart' + format="multipart", ) if hss.UPLOAD_ENABLED: self.assertEqual(rfs.HTTP_201_CREATED, response.status_code) - response = c.get('/api/v1/tilesets/') + response = c.get("/api/v1/tilesets/") - obj = tm.Tileset.objects.get(uuid='bb') + obj = tm.Tileset.objects.get(uuid="bb") # make sure the file was actually created self.assertTrue(op.exists, obj.datafile.path) else: self.assertEqual(403, response.status_code) - def test_link_file(self): - ''' + """ Linking a file assuems the file is already accessable to the system, i.e. with somthing like goofys, mounting s3 buckets - ''' + """ c = dt.Client() - c.login(username='user1', password='pass') - add_file('data/Dixon2012-J1-NcoI-R1-filtered.100kb.multires.cool', sub_dir='.') + c.login(username="user1", password="pass") + add_file("data/Dixon2012-J1-NcoI-R1-filtered.100kb.multires.cool", sub_dir=".") response = c.post( - '/api/v1/link_tile/', - json.dumps({ - 'filepath': 'Dixon2012-J1-NcoI-R1-filtered.100kb.multires.cool', - 'filetype': 'hitile', - 'datatype': 'vector', - 'uuid': 'bb', - 'private': 'True', - 'coordSystem': 'hg19' - }), + "/api/v1/link_tile/", + json.dumps( + { + "filepath": "Dixon2012-J1-NcoI-R1-filtered.100kb.multires.cool", + "filetype": "hitile", + "datatype": "vector", + "uuid": "bb", + "private": "True", + "coordSystem": "hg19", + } + ), content_type="application/json", ) - if hss.UPLOAD_ENABLED: self.assertEqual(rfs.HTTP_201_CREATED, response.status_code) - response = c.get('/api/v1/tilesets/') + response = c.get("/api/v1/tilesets/") - new_uuid = response.json()['results'][0]['uuid'] + new_uuid = response.json()["results"][0]["uuid"] obj = tm.Tileset.objects.get(uuid=new_uuid) # make sure the file was actually created @@ -1372,20 +1343,18 @@ def test_link_file(self): self.assertEqual(403, response.status_code) def test_register_url(self): - ''' + """ Registering a url allows the file to remain on a remote server and be accessed through the local higlass-server - ''' + """ c = dt.Client() - c.login(username='user1', password='pass') + c.login(username="user1", password="pass") - upload_file = open('data/chromSizes.tsv', 'rb') + upload_file = open("data/chromSizes.tsv", "rb") mv = tm.Tileset.objects.create( - datafile=dcfu.SimpleUploadedFile( - upload_file.name, upload_file.read() - ), - filetype='chromsizes-tsv', - datatype='chromsizes', + datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), + filetype="chromsizes-tsv", + datatype="chromsizes", coordSystem="hg19_url", ) @@ -1393,223 +1362,244 @@ def test_register_url(self): url = "https://s3.amazonaws.com/pkerp/public/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig" response = c.post( - '/api/v1/register_url/', - json.dumps({ - "fileurl": url, - "uid": uid, - "name": uid, - "datatype": "vector", - "filetype": "bigwig", - "coordSystem": "hg19_url" - }), + "/api/v1/register_url/", + json.dumps( + { + "fileurl": url, + "uid": uid, + "name": uid, + "datatype": "vector", + "filetype": "bigwig", + "coordSystem": "hg19_url", + } + ), content_type="application/json", ) self.assertEqual(200, response.status_code, response.content) - response = c.get('/api/v1/tilesets/') - record_matches = [result for result in response.json()['results'] if result['uuid'] == uid] + response = c.get("/api/v1/tilesets/") + record_matches = [ + result for result in response.json()["results"] if result["uuid"] == uid + ] assert len(record_matches) == 1 - assert record_matches[0]['name'] == uid + assert record_matches[0]["name"] == uid obj = tm.Tileset.objects.get(uuid=uid) assert uid in obj.datafile.path - response = c.get('/api/v1/tileset_info/?d={uuid}'.format(uuid=uid)) + response = c.get("/api/v1/tileset_info/?d={uuid}".format(uuid=uid)) assert response.status_code == 200 - response = c.get('/api/v1/tiles/?d={uuid}.0.0.0'.format(uuid=uid)) + response = c.get("/api/v1/tiles/?d={uuid}.0.0.0".format(uuid=uid)) assert response.status_code == 200 class GetterTest(dt.TestCase): def test_get_info(self): - filepath = 'data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool' + filepath = "data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool" info = hgco.tileset_info(filepath) - self.assertEqual(info['max_zoom'], 4) - self.assertEqual(info['max_width'], 1000000 * 2 ** 12) + self.assertEqual(info["max_zoom"], 4) + self.assertEqual(info["max_width"], 1000000 * 2 ** 12) class Bed2DDBTest(dt.TestCase): def setUp(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") - upload_file = open('data/arrowhead_domains_short.txt.multires.db', 'rb') - #x = upload_file.read() + upload_file = open("data/arrowhead_domains_short.txt.multires.db", "rb") + # x = upload_file.read() self.tileset = tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), - filetype='bed2ddb', - datatype='arrowhead-domains', + filetype="bed2ddb", + datatype="arrowhead-domains", owner=self.user1, - uuid='ahd') - + uuid="ahd", + ) - upload_file1 = open('data/Rao_RepH_GM12878_InsulationScore.txt.multires.db', 'rb') + upload_file1 = open( + "data/Rao_RepH_GM12878_InsulationScore.txt.multires.db", "rb" + ) self.tileset1 = tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file1.name, upload_file1.read()), - filetype='bed2ddb', - datatype='2d-rectangle-domains', + filetype="bed2ddb", + datatype="2d-rectangle-domains", owner=self.user1, - uuid='ahe') + uuid="ahe", + ) def test_uids_by_filename(self): - ret = self.client.get('/api/v1/uids_by_filename/?d=Rao_RepH_GM12878_InsulationScore.txt') + ret = self.client.get( + "/api/v1/uids_by_filename/?d=Rao_RepH_GM12878_InsulationScore.txt" + ) - contents = json.loads(ret.content.decode('utf-8')) + contents = json.loads(ret.content.decode("utf-8")) - assert(contents["count"] == 1) + assert contents["count"] == 1 - ret = self.client.get('/api/v1/uids_by_filename/?d=xRao_RepH_GM12878_InsulationScore') + ret = self.client.get( + "/api/v1/uids_by_filename/?d=xRao_RepH_GM12878_InsulationScore" + ) - contents = json.loads(ret.content.decode('utf-8')) + contents = json.loads(ret.content.decode("utf-8")) - assert(contents["count"] == 0) + assert contents["count"] == 0 def test_get_tile(self): - tile_id="{uuid}.{z}.{x}.{y}".format(uuid=self.tileset.uuid, z=0, x=0, y=0) - returned_text = self.client.get('/api/v1/tiles/?d={tile_id}'.format(tile_id=tile_id)) - returned = json.loads(returned_text.content.decode('utf-8')) + tile_id = "{uuid}.{z}.{x}.{y}".format(uuid=self.tileset.uuid, z=0, x=0, y=0) + returned_text = self.client.get( + "/api/v1/tiles/?d={tile_id}".format(tile_id=tile_id) + ) + returned = json.loads(returned_text.content.decode("utf-8")) - ret = self.client.get('/api/v1/tiles/?d={}.0.0.0'.format(self.tileset1.uuid)) - assert(ret.status_code == 200) + ret = self.client.get("/api/v1/tiles/?d={}.0.0.0".format(self.tileset1.uuid)) + assert ret.status_code == 200 - contents = json.loads(ret.content.decode('utf-8')) + contents = json.loads(ret.content.decode("utf-8")) def test_get_tiles(self): - tile_id00="{uuid}.{z}.{x}.{y}".format(uuid=self.tileset.uuid, z=0, x=0, y=0) - tile_id01="{uuid}.{z}.{x}.{y}".format(uuid=self.tileset.uuid, z=0, x=0, y=1) - tile_id10="{uuid}.{z}.{x}.{y}".format(uuid=self.tileset.uuid, z=0, x=1, y=0) - returned_text = self.client.get('/api/v1/tiles/?d={}&d={}&d={}'.format(tile_id00, tile_id01, tile_id10)) - returned = json.loads(returned_text.content.decode('utf-8')) + tile_id00 = "{uuid}.{z}.{x}.{y}".format(uuid=self.tileset.uuid, z=0, x=0, y=0) + tile_id01 = "{uuid}.{z}.{x}.{y}".format(uuid=self.tileset.uuid, z=0, x=0, y=1) + tile_id10 = "{uuid}.{z}.{x}.{y}".format(uuid=self.tileset.uuid, z=0, x=1, y=0) + returned_text = self.client.get( + "/api/v1/tiles/?d={}&d={}&d={}".format(tile_id00, tile_id01, tile_id10) + ) + returned = json.loads(returned_text.content.decode("utf-8")) - ret = self.client.get('/api/v1/tiles/?d={}.0.0.0'.format(self.tileset1.uuid)) - assert(ret.status_code == 200) + ret = self.client.get("/api/v1/tiles/?d={}.0.0.0".format(self.tileset1.uuid)) + assert ret.status_code == 200 - contents = json.loads(ret.content.decode('utf-8')) + contents = json.loads(ret.content.decode("utf-8")) def test_get_info(self): - ret = self.client.get('/api/v1/tileset_info/?d={}'.format(self.tileset1.uuid)) + ret = self.client.get("/api/v1/tileset_info/?d={}".format(self.tileset1.uuid)) - assert(ret.status_code == 200) + assert ret.status_code == 200 class BedDBTest(dt.TestCase): def setUp(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") - upload_file = open('data/gene_annotations.short.db', 'rb') - #x = upload_file.read() + upload_file = open("data/gene_annotations.short.db", "rb") + # x = upload_file.read() self.tileset = tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), - filetype='beddb', - datatype='gene-annotations', + filetype="beddb", + datatype="gene-annotations", owner=self.user1, - uuid='bdb') - + uuid="bdb", + ) def test_get_info(self): - upload_file = open('data/984627_PM16-00568-A_SM-9J5GB.beddb', 'rb') + upload_file = open("data/984627_PM16-00568-A_SM-9J5GB.beddb", "rb") tileset1 = tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), - filetype='beddb', - datatype='bedlike', + filetype="beddb", + datatype="bedlike", owner=self.user1, - uuid='bdc') - returned_text = self.client.get('/api/v1/tileset_info/?d=bdc') - returned = json.loads(returned_text.content.decode('utf-8')) + uuid="bdc", + ) + returned_text = self.client.get("/api/v1/tileset_info/?d=bdc") + returned = json.loads(returned_text.content.decode("utf-8")) def test_get_tile(self): - tile_id="{uuid}.{z}.{x}".format(uuid=self.tileset.uuid, z=0, x=0) - returned_text = self.client.get('/api/v1/tiles/?d={tile_id}'.format(tile_id=tile_id)) - returned = json.loads(returned_text.content.decode('utf-8')) + tile_id = "{uuid}.{z}.{x}".format(uuid=self.tileset.uuid, z=0, x=0) + returned_text = self.client.get( + "/api/v1/tiles/?d={tile_id}".format(tile_id=tile_id) + ) + returned = json.loads(returned_text.content.decode("utf-8")) - for x in returned['bdb.0.0']: - assert('uid' in x) - assert('importance' in x) - assert('fields' in x) + for x in returned["bdb.0.0"]: + assert "uid" in x + assert "importance" in x + assert "fields" in x def test_get_tiles(self): - tile_id="{uuid}.{z}.{x}".format(uuid=self.tileset.uuid, z=1, x=0) - tile_id1="{uuid}.{z}.{x}".format(uuid=self.tileset.uuid, z=1, x=1) - returned_text = self.client.get('/api/v1/tiles/?d={tile_id}&d={tile_id1}'.format(tile_id=tile_id, tile_id1=tile_id1)) - returned = json.loads(returned_text.content.decode('utf-8')) + tile_id = "{uuid}.{z}.{x}".format(uuid=self.tileset.uuid, z=1, x=0) + tile_id1 = "{uuid}.{z}.{x}".format(uuid=self.tileset.uuid, z=1, x=1) + returned_text = self.client.get( + "/api/v1/tiles/?d={tile_id}&d={tile_id1}".format( + tile_id=tile_id, tile_id1=tile_id1 + ) + ) + returned = json.loads(returned_text.content.decode("utf-8")) - assert(len(returned[tile_id]) > 0) - assert(len(returned[tile_id1]) > 0) + assert len(returned[tile_id]) > 0 + assert len(returned[tile_id1]) > 0 for x in returned[tile_id]: - assert('uid' in x) - assert('importance' in x) - assert('fields' in x) + assert "uid" in x + assert "importance" in x + assert "fields" in x for x in returned[tile_id1]: - assert('uid' in x) - assert('importance' in x) - assert('fields' in x) + assert "uid" in x + assert "importance" in x + assert "fields" in x class HiBedTest(dt.TestCase): - ''' + """ Test retrieving interval data (hibed) - ''' + """ + def setUp(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") - upload_file = open('data/cnv_short.hibed', 'rb') - #x = upload_file.read() + upload_file = open("data/cnv_short.hibed", "rb") + # x = upload_file.read() self.tileset = tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), - filetype='hibed', - datatype='stacked-interval', - coordSystem='hg19', + filetype="hibed", + datatype="stacked-interval", + coordSystem="hg19", owner=self.user1, - uuid='hbt') - + uuid="hbt", + ) def test_hibed_get_tile(self): - tile_id="{uuid}.{z}.{x}".format(uuid=self.tileset.uuid, z=0, x=0) - returned_text = self.client.get('/api/v1/tiles/?d={tile_id}'.format(tile_id=tile_id)) - returned = json.loads(returned_text.content.decode('utf-8')) + tile_id = "{uuid}.{z}.{x}".format(uuid=self.tileset.uuid, z=0, x=0) + returned_text = self.client.get( + "/api/v1/tiles/?d={tile_id}".format(tile_id=tile_id) + ) + returned = json.loads(returned_text.content.decode("utf-8")) - self.assertTrue('discrete' in returned[tile_id]) + self.assertTrue("discrete" in returned[tile_id]) def test_hibed_get_tileset_info(self): - tile_id="{uuid}".format(uuid=self.tileset.uuid) - returned_text = self.client.get('/api/v1/tileset_info/?d={tile_id}'.format(tile_id=tile_id)) - returned = json.loads(returned_text.content.decode('utf-8')) + tile_id = "{uuid}".format(uuid=self.tileset.uuid) + returned_text = self.client.get( + "/api/v1/tileset_info/?d={tile_id}".format(tile_id=tile_id) + ) + returned = json.loads(returned_text.content.decode("utf-8")) - self.assertTrue('tile_size' in returned[tile_id]) - self.assertEqual(returned[tile_id]['coordSystem'], 'hg19') + self.assertTrue("tile_size" in returned[tile_id]) + self.assertEqual(returned[tile_id]["coordSystem"], "hg19") class TilesetsViewSetTest(dt.TestCase): def setUp(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) - self.user2 = dcam.User.objects.create_user( - username='user2', password='pass' - ) + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") + self.user2 = dcam.User.objects.create_user(username="user2", password="pass") - upload_file = open('data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool', 'rb') + upload_file = open( + "data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool", "rb" + ) self.cooler = tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), - filetype='cooler', - owner=self.user1 + filetype="cooler", + owner=self.user1, ) - upload_file=open('data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile', 'rb') + upload_file = open( + "data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile", "rb" + ) self.hitile = tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), - filetype='hitile', - owner=self.user1 + filetype="hitile", + owner=self.user1, ) def tearDown(self): @@ -1618,13 +1608,15 @@ def tearDown(self): def check_tile(self, z, x, y): returned = json.loads( self.client.get( - '/api/v1/tiles/?d={uuid}.{z}.{x}.{y}'.format( + "/api/v1/tiles/?d={uuid}.{z}.{x}.{y}".format( uuid=self.cooler.uuid, x=x, y=y, z=z ) - ).content.decode('utf-8') + ).content.decode("utf-8") ) - r = base64.decodestring(returned[list(returned.keys())[0]]['dense'].encode('utf-8')) + r = base64.decodestring( + returned[list(returned.keys())[0]]["dense"].encode("utf-8") + ) q = np.frombuffer(r, dtype=np.float32) with h5py.File(self.cooler.datafile.path) as f: @@ -1637,9 +1629,9 @@ def check_tile(self, z, x, y): BINS_PER_TILE = 256 hdf_for_resolution = tileset_file[str(zoom_level)] - resolution = (tileset_info['max_width'] / 2**zoom_level) / BINS_PER_TILE + resolution = (tileset_info["max_width"] / 2 ** zoom_level) / BINS_PER_TILE - t = hgco.make_tiles(hdf_for_resolution, resolution, x, y)[(x,y)] + t = hgco.make_tiles(hdf_for_resolution, resolution, x, y)[(x, y)] q = q.astype(float) t = q.astype(float) @@ -1657,78 +1649,80 @@ def test_create_with_anonymous_user(self): Don't allow the creation of datasets by anonymous users. """ with self.assertRaises(ValueError): - upload_file =open('data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile', 'rb') + upload_file = open( + "data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile", + "rb", + ) tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), - filetype='hitile', - owner=dcam.AnonymousUser() + filetype="hitile", + owner=dcam.AnonymousUser(), ) def test_post_dataset(self): c = dt.Client() - c.login(username='user1', password='pass') - f = open('data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile', 'rb') + c.login(username="user1", password="pass") + f = open( + "data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile", "rb" + ) ret = c.post( - '/api/v1/tilesets/', + "/api/v1/tilesets/", { - 'datafile': f, - 'filetype': 'hitile', - 'private': 'True', - 'coordSystem': 'hg19' - } - , - format='multipart' + "datafile": f, + "filetype": "hitile", + "private": "True", + "coordSystem": "hg19", + }, + format="multipart", ) f.close() - ret_obj = json.loads(ret.content.decode('utf-8')) + ret_obj = json.loads(ret.content.decode("utf-8")) if hss.UPLOAD_ENABLED: - t = tm.Tileset.objects.get(uuid=ret_obj['uuid']) + t = tm.Tileset.objects.get(uuid=ret_obj["uuid"]) # this object should be private because we were logged in and # requested it to be private self.assertTrue(t.private) - c.login(username='user2', password='pass') - ret = c.get('/api/v1/tileset_info/?d={uuid}'.format( - uuid=ret_obj['uuid']) - ) + c.login(username="user2", password="pass") + ret = c.get("/api/v1/tileset_info/?d={uuid}".format(uuid=ret_obj["uuid"])) # user2 should not be able to get information about this dataset - ts_info = json.loads(ret.content.decode('utf-8')) - self.assertTrue('error' in ts_info[ret_obj['uuid']]) + ts_info = json.loads(ret.content.decode("utf-8")) + self.assertTrue("error" in ts_info[ret_obj["uuid"]]) - c.login(username='user1', password='pass') - ret = c.get('/api/v1/tileset_info/?d={uuid}'.format( - uuid=ret_obj['uuid']) - ) + c.login(username="user1", password="pass") + ret = c.get("/api/v1/tileset_info/?d={uuid}".format(uuid=ret_obj["uuid"])) # user1 should be able to access it - ts_info = json.loads(ret.content.decode('utf-8')) - self.assertFalse('error' in ts_info[ret_obj['uuid']]) - self.assertEqual(ts_info[ret_obj['uuid']]['coordSystem'], 'hg19') + ts_info = json.loads(ret.content.decode("utf-8")) + self.assertFalse("error" in ts_info[ret_obj["uuid"]]) + self.assertEqual(ts_info[ret_obj["uuid"]]["coordSystem"], "hg19") # upload a new dataset as user1 ret = c.post( - '/api/v1/tilesets/', + "/api/v1/tilesets/", { - 'datafile': open('data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile', 'rb'), - 'filetype': 'hitile', - 'private': 'False', - 'coordSystem': 'hg19' - + "datafile": open( + "data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile", + "rb", + ), + "filetype": "hitile", + "private": "False", + "coordSystem": "hg19", }, - format='multipart' + format="multipart", ) - ret_obj = json.loads(ret.content.decode('utf-8')) + ret_obj = json.loads(ret.content.decode("utf-8")) # since the previously uploaded dataset is not private, we should be # able to access it as user2 - c.login(username='user2', password='pass') - ret = c.get('/api/v1/tileset_info/?d={uuid}'.format(uuid=ret_obj['uuid'])) - ts_info = json.loads(ret.content.decode('utf-8')) + c.login(username="user2", password="pass") + ret = c.get("/api/v1/tileset_info/?d={uuid}".format(uuid=ret_obj["uuid"])) + ts_info = json.loads(ret.content.decode("utf-8")) - self.assertFalse('error' in ts_info[ret_obj['uuid']]) + self.assertFalse("error" in ts_info[ret_obj["uuid"]]) else: self.assertEquals(ret.status_code, 403) @@ -1737,20 +1731,22 @@ def test_create_private_tileset(self): access it if we're logged in as the proper user """ - upload_file =open('data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile', 'rb') + upload_file = open( + "data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile", "rb" + ) private_obj = tm.Tileset.objects.create( datafile=dcfu.SimpleUploadedFile(upload_file.name, upload_file.read()), - filetype='hitile', + filetype="hitile", private=True, - owner=self.user1 + owner=self.user1, ) c = dt.Client() - c.login(username='user1', password='pass') + c.login(username="user1", password="pass") returned = json.loads( self.client.get( - '/api/v1/tileset_info/?d={uuid}'.format(uuid=private_obj.uuid) - ).content.decode('utf-8') + "/api/v1/tileset_info/?d={uuid}".format(uuid=private_obj.uuid) + ).content.decode("utf-8") ) def test_get_top_tile(self): @@ -1768,18 +1764,14 @@ def test_get_many_tiles(self): returned = json.loads( self.client.get( - '/api/v1/tiles/?d={uuid}.1.0.0&d={uuid}.1.0.1'.format( + "/api/v1/tiles/?d={uuid}.1.0.0&d={uuid}.1.0.1".format( uuid=self.cooler.uuid ) - ).content.decode('utf-8') + ).content.decode("utf-8") ) - self.assertTrue('{uuid}.1.0.0'.format( - uuid=self.cooler.uuid) in returned.keys() - ) - self.assertTrue('{uuid}.1.0.1'.format( - uuid=self.cooler.uuid) in returned.keys() - ) + self.assertTrue("{uuid}.1.0.0".format(uuid=self.cooler.uuid) in returned.keys()) + self.assertTrue("{uuid}.1.0.1".format(uuid=self.cooler.uuid) in returned.keys()) def test_get_same_tiles(self): """ @@ -1787,10 +1779,10 @@ def test_get_same_tiles(self): """ returned = json.loads( self.client.get( - '/api/v1/tiles/?d={uuid}.1.0.0&d={uuid}.1.0.0'.format( + "/api/v1/tiles/?d={uuid}.1.0.0&d={uuid}.1.0.0".format( uuid=self.cooler.uuid ) - ).content.decode('utf-8') + ).content.decode("utf-8") ) self.assertEquals(len(returned.keys()), 1) @@ -1802,15 +1794,13 @@ def test_get_nonexistent_tile(self): non-existent tile. It just needs to be missing from the return array. """ - returned = json.loads( self.client.get( - '/api/v1/tiles/?d={uuid}.1.5.5'.format(uuid=self.cooler.uuid) - ).content.decode('utf-8') + "/api/v1/tiles/?d={uuid}.1.5.5".format(uuid=self.cooler.uuid) + ).content.decode("utf-8") ) - - ''' + """ # Not sure that this should still be the case # now it returns an array of NaNs self.assertTrue( @@ -1818,53 +1808,48 @@ def test_get_nonexistent_tile(self): uuid=self.cooler.uuid ) not in returned.keys() ) - ''' + """ returned = json.loads( self.client.get( - '/api/v1/tiles/?d={uuid}.20.5.5'.format(uuid=self.cooler.uuid) - ).content.decode('utf-8') + "/api/v1/tiles/?d={uuid}.20.5.5".format(uuid=self.cooler.uuid) + ).content.decode("utf-8") ) self.assertTrue( - '{uuid}.1.5.5'.format( - uuid=self.cooler.uuid - ) not in returned.keys() + "{uuid}.1.5.5".format(uuid=self.cooler.uuid) not in returned.keys() ) def test_get_hitile_tileset_info(self): returned = json.loads( self.client.get( - '/api/v1/tileset_info/?d={uuid}'.format(uuid=self.hitile.uuid) - ).content.decode('utf-8') + "/api/v1/tileset_info/?d={uuid}".format(uuid=self.hitile.uuid) + ).content.decode("utf-8") ) uuid = "{uuid}".format(uuid=self.hitile.uuid) - self.assertTrue( - "{uuid}".format(uuid=self.hitile.uuid) in returned.keys() - ) - self.assertEqual(returned[uuid][u'max_zoom'], 22) - self.assertEqual(returned[uuid][u'max_width'], 2 ** 32) + self.assertTrue("{uuid}".format(uuid=self.hitile.uuid) in returned.keys()) + self.assertEqual(returned[uuid]["max_zoom"], 22) + self.assertEqual(returned[uuid]["max_width"], 2 ** 32) - self.assertTrue(u'name' in returned[uuid]) + self.assertTrue("name" in returned[uuid]) def test_get_cooler_tileset_info(self): returned = json.loads( self.client.get( - '/api/v1/tileset_info/?d={uuid}'.format(uuid=self.cooler.uuid) - ).content.decode('utf-8') + "/api/v1/tileset_info/?d={uuid}".format(uuid=self.cooler.uuid) + ).content.decode("utf-8") ) uuid = "{uuid}".format(uuid=self.cooler.uuid) - self.assertTrue(u'name' in returned[uuid]) - + self.assertTrue("name" in returned[uuid]) def test_get_hitile_tile(self): returned = json.loads( self.client.get( - '/api/v1/tiles/?d={uuid}.0.0'.format(uuid=self.hitile.uuid) - ).content.decode('utf-8') + "/api/v1/tiles/?d={uuid}.0.0".format(uuid=self.hitile.uuid) + ).content.decode("utf-8") ) self.assertTrue("{uuid}.0.0".format(uuid=self.hitile.uuid) in returned) @@ -1872,179 +1857,218 @@ def test_get_hitile_tile(self): def test_list_tilesets(self): c = dt.Client() - c.login(username='user1', password='pass') + c.login(username="user1", password="pass") ret = c.post( - '/api/v1/tilesets/', + "/api/v1/tilesets/", { - 'datafile': open('data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile','rb'), - 'filetype': 'hitile', - 'private': 'True', - 'name': 'one', - 'coordSystem': 'hg19' - } + "datafile": open( + "data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile", + "rb", + ), + "filetype": "hitile", + "private": "True", + "name": "one", + "coordSystem": "hg19", + }, ) ret = c.post( - '/api/v1/tilesets/', + "/api/v1/tilesets/", { - 'datafile': open('data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile','rb'), - 'filetype': 'hitile', - 'private': 'True', - 'name': 'tone', - 'coordSystem': 'hg19' - } + "datafile": open( + "data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile", + "rb", + ), + "filetype": "hitile", + "private": "True", + "name": "tone", + "coordSystem": "hg19", + }, ) ret = c.post( - '/api/v1/tilesets/', + "/api/v1/tilesets/", { - 'datafile': open('data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile','rb'), - 'filetype': 'cooler', - 'private': 'True', - 'name': 'tax', - 'coordSystem': 'hg19' - } - ) - ret = c.get('/api/v1/tilesets/?ac=ne') + "datafile": open( + "data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile", + "rb", + ), + "filetype": "cooler", + "private": "True", + "name": "tax", + "coordSystem": "hg19", + }, + ) + ret = c.get("/api/v1/tilesets/?ac=ne") self.assertEquals(ret.status_code, 200) - ret = json.loads(ret.content.decode('utf-8')) + ret = json.loads(ret.content.decode("utf-8")) if hss.UPLOAD_ENABLED: - count1 = ret['count'] + count1 = ret["count"] self.assertTrue(count1 > 0) - names = set([ts['name'] for ts in ret['results']]) + names = set([ts["name"] for ts in ret["results"]]) - self.assertTrue(u'one' in names) - self.assertFalse(u'tax' in names) + self.assertTrue("one" in names) + self.assertFalse("tax" in names) c.logout() # all tilesets should be private - ret = json.loads(c.get('/api/v1/tilesets/?ac=ne').content.decode('utf-8')) - self.assertEquals(ret['count'], 0) + ret = json.loads(c.get("/api/v1/tilesets/?ac=ne").content.decode("utf-8")) + self.assertEquals(ret["count"], 0) - ret = json.loads(c.get('/api/v1/tilesets/?ac=ne&t=cooler').content.decode('utf-8')) - count1 = ret['count'] + ret = json.loads( + c.get("/api/v1/tilesets/?ac=ne&t=cooler").content.decode("utf-8") + ) + count1 = ret["count"] self.assertTrue(count1 == 0) - c.login(username='user2', password='pass') - ret = json.loads(c.get('/api/v1/tilesets/?q=ne').content.decode('utf-8')) + c.login(username="user2", password="pass") + ret = json.loads(c.get("/api/v1/tilesets/?q=ne").content.decode("utf-8")) - names = set([ts['name'] for ts in ret['results']]) - self.assertFalse(u'one' in names) + names = set([ts["name"] for ts in ret["results"]]) + self.assertFalse("one" in names) ret = c.post( - '/api/v1/tilesets/', + "/api/v1/tilesets/", { - 'datafile': open('data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile','rb'), - 'filetype': 'xxxyx', - 'datatype': 'vector', - 'private': 'True', - } + "datafile": open( + "data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile", + "rb", + ), + "filetype": "xxxyx", + "datatype": "vector", + "private": "True", + }, ) # not coordSystem field - assert(ret.status_code == rfs.HTTP_400_BAD_REQUEST) - ret = json.loads(c.get('/api/v1/tilesets/?t=xxxyx').content.decode('utf-8')) + assert ret.status_code == rfs.HTTP_400_BAD_REQUEST + ret = json.loads(c.get("/api/v1/tilesets/?t=xxxyx").content.decode("utf-8")) - assert(ret['count'] == 0) + assert ret["count"] == 0 ret = c.post( - '/api/v1/tilesets/', + "/api/v1/tilesets/", { - 'datafile': open('data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile','rb'), - 'filetype': 'xxxyx', - 'datatype': 'vector', - 'private': 'True', - 'coordSystem': 'hg19', - } + "datafile": open( + "data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile", + "rb", + ), + "filetype": "xxxyx", + "datatype": "vector", + "private": "True", + "coordSystem": "hg19", + }, ) - ret = json.loads(c.get('/api/v1/tilesets/?t=xxxyx').content.decode('utf-8')) - self.assertEqual(ret['count'], 1) + ret = json.loads(c.get("/api/v1/tilesets/?t=xxxyx").content.decode("utf-8")) + self.assertEqual(ret["count"], 1) else: - self.assertEquals(ret['count'], 0) + self.assertEquals(ret["count"], 0) def test_add_with_uid(self): - self.client.login(username='user1', password='pass') + self.client.login(username="user1", password="pass") ret = self.client.post( - '/api/v1/tilesets/', + "/api/v1/tilesets/", { - 'datafile': open('data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile','rb'), - 'filetype': 'a', - 'datatype': 'vector', - 'private': 'True', - 'uid': 'aaaaaaaaaaaaaaaaaaaaaa', - 'coordSystem': 'hg19' - } + "datafile": open( + "data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile", + "rb", + ), + "filetype": "a", + "datatype": "vector", + "private": "True", + "uid": "aaaaaaaaaaaaaaaaaaaaaa", + "coordSystem": "hg19", + }, ) - ret = json.loads(self.client.get('/api/v1/tilesets/').content.decode('utf-8')) + ret = json.loads(self.client.get("/api/v1/tilesets/").content.decode("utf-8")) if hss.UPLOAD_ENABLED: # the two default datasets plus the added one - self.assertEquals(ret['count'], 3) + self.assertEquals(ret["count"], 3) # try to add one more dataset with a specified uid ret = json.loads( self.client.post( - '/api/v1/tilesets/', + "/api/v1/tilesets/", { - 'datafile': open('data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile','rb'), - 'filetype': 'a', - 'datatype': 'vector', - 'private': 'True', - 'uid': 'aaaaaaaaaaaaaaaaaaaaaa', - 'coordSystem': 'hg19' - } - ).content.decode('utf-8') + "datafile": open( + "data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile", + "rb", + ), + "filetype": "a", + "datatype": "vector", + "private": "True", + "uid": "aaaaaaaaaaaaaaaaaaaaaa", + "coordSystem": "hg19", + }, + ).content.decode("utf-8") ) # there should be a return value explaining that we can't add a tileset # which has an existing uuid - self.assertTrue('detail' in ret) + self.assertTrue("detail" in ret) - ret = json.loads(self.client.get('/api/v1/tilesets/').content.decode('utf-8')) - self.assertEquals(ret['count'], 3) + ret = json.loads( + self.client.get("/api/v1/tilesets/").content.decode("utf-8") + ) + self.assertEquals(ret["count"], 3) else: - self.assertEquals(ret['count'], 2) + self.assertEquals(ret["count"], 2) def test_list_by_datatype(self): - self.client.login(username='user1', password='pass') + self.client.login(username="user1", password="pass") ret = self.client.post( - '/api/v1/tilesets/', + "/api/v1/tilesets/", { - 'datafile': open('data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile','rb'), - 'filetype': 'a', - 'datatype': '1', - 'private': 'True', - 'coordSystem': 'hg19', - 'uid': 'aaaaaaaaaaaaaaaaaaaaaa' - } + "datafile": open( + "data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile", + "rb", + ), + "filetype": "a", + "datatype": "1", + "private": "True", + "coordSystem": "hg19", + "uid": "aaaaaaaaaaaaaaaaaaaaaa", + }, ) ret = self.client.post( - '/api/v1/tilesets/', + "/api/v1/tilesets/", { - 'datafile': open('data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile','rb'), - 'filetype': 'a', - 'datatype': '2', - 'private': 'True', - 'coordSystem': 'hg19', - 'uid': 'bb' - } + "datafile": open( + "data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile", + "rb", + ), + "filetype": "a", + "datatype": "2", + "private": "True", + "coordSystem": "hg19", + "uid": "bb", + }, ) - ret = json.loads(self.client.get('/api/v1/tilesets/?dt=1').content.decode('utf-8')) - self.assertEqual(ret['count'], 1 if hss.UPLOAD_ENABLED else 0) + ret = json.loads( + self.client.get("/api/v1/tilesets/?dt=1").content.decode("utf-8") + ) + self.assertEqual(ret["count"], 1 if hss.UPLOAD_ENABLED else 0) - ret = json.loads(self.client.get('/api/v1/tilesets/?dt=2').content.decode('utf-8')) - self.assertEqual(ret['count'], 1 if hss.UPLOAD_ENABLED else 0) + ret = json.loads( + self.client.get("/api/v1/tilesets/?dt=2").content.decode("utf-8") + ) + self.assertEqual(ret["count"], 1 if hss.UPLOAD_ENABLED else 0) - ret = json.loads(self.client.get('/api/v1/tilesets/?dt=1&dt=2').content.decode('utf-8')) - self.assertEqual(ret['count'], 2 if hss.UPLOAD_ENABLED else 0) + ret = json.loads( + self.client.get("/api/v1/tilesets/?dt=1&dt=2").content.decode("utf-8") + ) + self.assertEqual(ret["count"], 2 if hss.UPLOAD_ENABLED else 0) def test_get_nonexistant_tileset_info(self): - ret = json.loads(self.client.get('/api/v1/tileset_info/?d=x1x').content.decode('utf-8')) + ret = json.loads( + self.client.get("/api/v1/tileset_info/?d=x1x").content.decode("utf-8") + ) # make sure above doesn't raise an error diff --git a/tilesets/urls.py b/tilesets/urls.py index cc227637..b4765fda 100644 --- a/tilesets/urls.py +++ b/tilesets/urls.py @@ -3,29 +3,29 @@ from rest_framework.routers import SimpleRouter from rest_framework_swagger.views import get_swagger_view -schema_view = get_swagger_view(title='Pastebin API') +schema_view = get_swagger_view(title="Pastebin API") # Create a router and register our viewsets with it. router = SimpleRouter() -router.register(r'tilesets', views.TilesetsViewSet, 'tilesets') -#router.register(r'users', views.UserViewSet) +router.register(r"tilesets", views.TilesetsViewSet, "tilesets") +# router.register(r'users', views.UserViewSet) # The API URLs are now determined automatically by the router. # Additionally, we include the login URLs for the browsable API. urlpatterns = [ - #url(r'^schema', schema_view), - url(r'^viewconf', views.viewconfs), - url(r'^uids_by_filename', views.uids_by_filename), - url(r'^tiles/$', views.tiles), - url(r'^tileset_info/$', views.tileset_info), - url(r'^suggest/$', views.suggest), - url(r'^', include(router.urls)), - url(r'^link_tile/$', views.link_tile), - url(r'^register_url/$', views.register_url), - url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')), - url(r'^chrom-sizes/$', views.sizes), - url(r'^available-chrom-sizes/$', views.available_chrom_sizes) - #url(r'^users/$', views.UserList.as_view()), - #url(r'^users/(?P[0-9]+)/$', views.UserDetail.as_view()) + # url(r'^schema', schema_view), + url(r"^viewconf", views.viewconfs), + url(r"^uids_by_filename", views.uids_by_filename), + url(r"^tiles/$", views.tiles), + url(r"^tileset_info/$", views.tileset_info), + url(r"^suggest/$", views.suggest), + url(r"^", include(router.urls)), + url(r"^link_tile/$", views.link_tile), + url(r"^register_url/$", views.register_url), + url(r"^api-auth/", include("rest_framework.urls", namespace="rest_framework")), + url(r"^chrom-sizes/$", views.sizes), + url(r"^available-chrom-sizes/$", views.available_chrom_sizes) + # url(r'^users/$', views.UserList.as_view()), + # url(r'^users/(?P[0-9]+)/$', views.UserDetail.as_view()) ] diff --git a/tilesets/views.py b/tilesets/views.py index f30b3de9..ae686cb1 100644 --- a/tilesets/views.py +++ b/tilesets/views.py @@ -85,23 +85,23 @@ class UserDetail(generics.RetrieveAPIView): serializer_class = tss.UserSerializer -@api_view(['GET']) +@api_view(["GET"]) def uids_by_filename(request): - ''' + """ Retrieve a list uids corresponding to a given filename - ''' + """ queryset = tm.Tileset.objects.all() - queryset = queryset.filter(datafile__contains=request.GET['d']) + queryset = queryset.filter(datafile__contains=request.GET["d"]) serializer = tss.UserFacingTilesetSerializer(queryset, many=True) return JsonResponse({"count": len(queryset), "results": serializer.data}) -@api_view(['GET']) +@api_view(["GET"]) @authentication_classes((CsrfExemptSessionAuthentication, BasicAuthentication)) def available_chrom_sizes(request): - ''' + """ Get the list of available chromosome size lists. Args: @@ -109,7 +109,7 @@ def available_chrom_sizes(request): Returns: A JSON response containing the list of chromosome size lists. - ''' + """ queryset = tm.Tileset.objects.all() queryset = queryset.filter(datatype__in=["chromsizes"]) @@ -118,10 +118,10 @@ def available_chrom_sizes(request): return JsonResponse({"count": len(queryset), "results": serializer.data}) -@api_view(['GET']) +@api_view(["GET"]) @authentication_classes((CsrfExemptSessionAuthentication, BasicAuthentication)) def sizes(request): - '''Return chromosome sizes. + """Return chromosome sizes. Retrieves the chromSiyes.tsv and either retrieves it as is or converts it to a JSON format. Args: @@ -152,22 +152,23 @@ def sizes(request): ... } ``` - ''' - uuid = request.GET.get('id', False) - res_type = request.GET.get('type', 'tsv') - incl_cum = request.GET.get('cum', False) + """ + uuid = request.GET.get("id", False) + res_type = request.GET.get("type", "tsv") + incl_cum = request.GET.get("cum", False) response = HttpResponse is_json = False - if res_type == 'json': + if res_type == "json": is_json = True response = JsonResponse - if res_type != 'json' and incl_cum: + if res_type != "json" and incl_cum: return response( - 'Sorry buddy. Cumulative sizes not yet supported for non-JSON ' - 'file types. 😞', status=501 + "Sorry buddy. Cumulative sizes not yet supported for non-JSON " + "file types. 😞", + status=501, ) # Try to find the db entry @@ -175,29 +176,29 @@ def sizes(request): chrom_sizes = tm.Tileset.objects.get(uuid=uuid) except Exception as e: logger.exception(e) - err_msg = 'Oh lord! ChromSizes for %s not found. 😬' % uuid + err_msg = "Oh lord! ChromSizes for %s not found. 😬" % uuid err_status = 404 if is_json: - return response({'error': err_msg}, status=err_status) + return response({"error": err_msg}, status=err_status) return response(err_msg, status=err_status) # Try to load the chromosome sizes and return them as a list of # (name, size) tuples try: - if tgt.get_tileset_filetype(chrom_sizes) == 'bigwig': + if tgt.get_tileset_filetype(chrom_sizes) == "bigwig": data = hgbi.chromsizes(chrom_sizes.datafile.path) - elif tgt.get_tileset_filetype(chrom_sizes) == 'bigbed': + elif tgt.get_tileset_filetype(chrom_sizes) == "bigbed": data = hgbb.chromsizes(chrom_sizes.datafile.path) - elif tgt.get_tileset_filetype(chrom_sizes) == 'cooler': + elif tgt.get_tileset_filetype(chrom_sizes) == "cooler": data = tcs.get_cooler_chromsizes(chrom_sizes.datafile.path) - elif tgt.get_tileset_filetype(chrom_sizes) == 'chromsizes-tsv': + elif tgt.get_tileset_filetype(chrom_sizes) == "chromsizes-tsv": data = tcs.get_tsv_chromsizes(chrom_sizes.datafile.path) - elif tgt.get_tileset_filetype(chrom_sizes) == 'multivec': + elif tgt.get_tileset_filetype(chrom_sizes) == "multivec": data = tcs.get_multivec_chromsizes(chrom_sizes.datafile.path) else: - data = ''; + data = "" except Exception as ex: logger.exception(ex) @@ -205,7 +206,7 @@ def sizes(request): err_status = 500 if is_json: - return response({'error': err_msg}, status=err_status) + return response({"error": err_msg}, status=err_status) return response(err_msg, status=err_status) @@ -214,72 +215,66 @@ def sizes(request): # data should be a list of (name, size) tuples coming # coming and converted to a more appropriate data type # going out - if res_type == 'tsv': + if res_type == "tsv": lines = [] for (name, size) in data: lines += ["{}\t{}\n".format(name, size)] data = lines - if res_type == 'json' and not incl_cum: + if res_type == "json" and not incl_cum: json_out = {} for row in data: - json_out[row[0]] = { - 'size': int(row[1]) - } + json_out[row[0]] = {"size": int(row[1])} data = json_out - if res_type == 'json' and incl_cum: + if res_type == "json" and incl_cum: json_out = {} cum = 0 for row in data: size = int(row[1]) - json_out[row[0]] = { - 'size': size, - 'offset': cum - } + json_out[row[0]] = {"size": size, "offset": cum} cum += size data = json_out except Exception as e: logger.exception(e) - err_msg = 'THIS IS AN OUTRAGE!!!1! Something failed. 😡' + err_msg = "THIS IS AN OUTRAGE!!!1! Something failed. 😡" err_status = 500 if is_json: - return response({'error': err_msg}, status=err_status) + return response({"error": err_msg}, status=err_status) return response(err_msg, status=err_status) return response(data) -@api_view(['GET']) + +@api_view(["GET"]) def suggest(request): - ''' + """ Suggest gene names based on the input text - ''' + """ # suggestions are taken from a gene annotations tileset - tileset_uuid = request.GET['d'] - text = request.GET['ac'] + tileset_uuid = request.GET["d"] + text = request.GET["ac"] try: tileset = tm.Tileset.objects.get(uuid=tileset_uuid) except ObjectDoesNotExist: - raise rfe.NotFound('Suggestion source file not found') + raise rfe.NotFound("Suggestion source file not found") - result_dict = tsu.get_gene_suggestions( - tileset.datafile.path, text - ) + result_dict = tsu.get_gene_suggestions(tileset.datafile.path, text) return JsonResponse(result_dict, safe=False) -@api_view(['GET', 'POST']) +@api_view(["GET", "POST"]) def viewconfs(request): - ''' + """ Retrieve a viewconfs with a given uid Args: @@ -289,71 +284,60 @@ def viewconfs(request): Return: - ''' - if request.method == 'POST': + """ + if request.method == "POST": if not hss.UPLOAD_ENABLED: - return JsonResponse({ - 'error': 'Uploads disabled' - }, status=403) + return JsonResponse({"error": "Uploads disabled"}, status=403) if request.user.is_anonymous and not hss.PUBLIC_UPLOAD_ENABLED: - return JsonResponse({ - 'error': 'Public uploads disabled' - }, status=403) + return JsonResponse({"error": "Public uploads disabled"}, status=403) - viewconf_wrapper = json.loads(request.body.decode('utf-8')) - uid = viewconf_wrapper.get('uid') or slugid.nice() + viewconf_wrapper = json.loads(request.body.decode("utf-8")) + uid = viewconf_wrapper.get("uid") or slugid.nice() try: - viewconf = json.dumps(viewconf_wrapper['viewconf']) + viewconf = json.dumps(viewconf_wrapper["viewconf"]) except KeyError: - return JsonResponse({ - 'error': 'Broken view config' - }, status=400) + return JsonResponse({"error": "Broken view config"}, status=400) try: - higlass_version = viewconf_wrapper['higlassVersion'] + higlass_version = viewconf_wrapper["higlassVersion"] except KeyError: - higlass_version = '' + higlass_version = "" existing_object = tm.ViewConf.objects.filter(uuid=uid) if len(existing_object) > 0: - return JsonResponse({ - 'error': 'Object with uid {} already exists'.format(uid) - }, status=rfs.HTTP_400_BAD_REQUEST); + return JsonResponse( + {"error": "Object with uid {} already exists".format(uid)}, + status=rfs.HTTP_400_BAD_REQUEST, + ) - serializer = tss.ViewConfSerializer(data={'viewconf': viewconf}) + serializer = tss.ViewConfSerializer(data={"viewconf": viewconf}) if not serializer.is_valid(): - return JsonResponse({ - 'error': 'Serializer not valid' - }, status=rfs.HTTP_400_BAD_REQUEST) + return JsonResponse( + {"error": "Serializer not valid"}, status=rfs.HTTP_400_BAD_REQUEST + ) - serializer.save( - uuid=uid, viewconf=viewconf, higlassVersion=higlass_version - ) + serializer.save(uuid=uid, viewconf=viewconf, higlassVersion=higlass_version) - return JsonResponse({'uid': uid}) + return JsonResponse({"uid": uid}) - uid = request.GET.get('d') + uid = request.GET.get("d") if not uid: - return JsonResponse({ - 'error': 'View config ID not specified' - }, status=404) + return JsonResponse({"error": "View config ID not specified"}, status=404) try: obj = tm.ViewConf.objects.get(uuid=uid) except ObjectDoesNotExist: - return JsonResponse({ - 'error': 'View config not found' - }, status=404) + return JsonResponse({"error": "View config not found"}, status=404) return JsonResponse(json.loads(obj.viewconf)) def add_transform_type(tile_id): - ''' + """ Add a transform type to a cooler tile id if it's not already present. @@ -366,8 +350,8 @@ def add_transform_type(tile_id): ------- new_tile_id: str A formatted tile id, potentially with an added transform_type - ''' - tile_id_parts = tile_id.split('.') + """ + tile_id_parts = tile_id.split(".") tileset_uuid = tile_id_parts[0] tile_position = tile_id_parts[1:4] @@ -376,9 +360,9 @@ def add_transform_type(tile_id): return new_tile_id -@api_view(['GET']) +@api_view(["GET"]) def tiles(request): - '''Retrieve a set of tiles + """Retrieve a set of tiles A call to this API function should retrieve a few tiles. @@ -392,19 +376,19 @@ def tiles(request): data being requested. The JSON object is just a dictionary of (tile_id, tile_data) items. - ''' + """ # create a set so that we don't fetch the same tile multiple times tileids_to_fetch = set(request.GET.getlist("d")) # with ProcessPoolExecutor() as executor: # res = executor.map(parallelize, hargs) - ''' + """ p = mp.Pool(4) res = p.map(parallelize, hargs) - ''' + """ # Return the raw data if only one tile is requested. This currently only # works for `imtiles` - raw = request.GET.get('raw', False) + raw = request.GET.get("raw", False) tileids_by_tileset = col.defaultdict(set) generated_tiles = [] @@ -423,7 +407,7 @@ def tiles(request): tileset = tm.Tileset.objects.get(uuid=tileset_uuid) tilesets[tileset_uuid] = tileset - if tileset.filetype == 'cooler': + if tileset.filetype == "cooler": # cooler tiles can have a transform (e.g. 'ice', 'kr') which # needs to be added if it's not there (e.g. 'default') new_tile_id = add_transform_type(tile_id) @@ -442,7 +426,7 @@ def tiles(request): # from the original data logger.warn(ex) - #tile_value = None + # tile_value = None if tile_value is not None: # we found the tile in the cache, no need to fetch it again @@ -454,13 +438,17 @@ def tiles(request): # fetch the tiles tilesets = [tilesets[tu] for tu in tileids_by_tileset] - accessible_tilesets = [(t, tileids_by_tileset[t.uuid], raw) for t in tilesets if ((not t.private) or request.user == t.owner)] + accessible_tilesets = [ + (t, tileids_by_tileset[t.uuid], raw) + for t in tilesets + if ((not t.private) or request.user == t.owner) + ] - #pool = mp.Pool(6) + # pool = mp.Pool(6) generated_tiles += list(it.chain(*map(tgt.generate_tiles, accessible_tilesets))) - ''' + """ for tileset_uuid in tileids_by_tileset: # load the tileset object tileset = tilesets[tileset_uuid] @@ -470,7 +458,7 @@ def tiles(request): generated_tiles += [(tile_id, {'error': "Forbidden"}) for tile_id in tileids_by_tileset[tileset_uuid]] else: generated_tiles += generate_tiles(tileset, tileids_by_tileset[tileset_uuid]) - ''' + """ # store the tiles in redis @@ -495,17 +483,15 @@ def tiles(request): if original_tile_id in tileids_to_fetch: tiles_to_return[original_tile_id] = tile_value - if len(generated_tiles) == 1 and raw and 'image' in generated_tiles[0][1]: - return HttpResponse( - generated_tiles[0][1]['image'], content_type='image/jpeg' - ) + if len(generated_tiles) == 1 and raw and "image" in generated_tiles[0][1]: + return HttpResponse(generated_tiles[0][1]["image"], content_type="image/jpeg") return JsonResponse(tiles_to_return, safe=False) -@api_view(['GET']) +@api_view(["GET"]) def tileset_info(request): - ''' Get information about a tileset + """ Get information about a tileset Tilesets have information critical to their display such as the maximum number of dimensions and well as @@ -518,149 +504,139 @@ def tileset_info(request): Return: django.http.JsonResponse: A JSON object containing the tileset meta-information - ''' + """ queryset = tm.Tileset.objects.all() tileset_uuids = request.GET.getlist("d") tileset_infos = {} chromsizes_error = None - if 'cs' in request.GET: + if "cs" in request.GET: # we need to call a different server to get the tiles - if not 'ci' in request.GET.getlist: - chromsizes_error = 'cs param present without ci' + if not "ci" in request.GET.getlist: + chromsizes_error = "cs param present without ci" # call the request server and get the chromsizes pass else: - if 'ci' in request.GET: + if "ci" in request.GET: try: - chromsizes = tm.Tileset.objects.get(uuid=request.GET['ci']) + chromsizes = tm.Tileset.objects.get(uuid=request.GET["ci"]) data = tcs.chromsizes_array_to_series( - tcs.get_tsv_chromsizes(chromsizes.datafile.path)) + tcs.get_tsv_chromsizes(chromsizes.datafile.path) + ) except Exception as ex: pass for tileset_uuid in tileset_uuids: tileset_object = queryset.filter(uuid=tileset_uuid).first() - if tileset_uuid == 'osm-image': + if tileset_uuid == "osm-image": tileset_infos[tileset_uuid] = { - 'min_x': 0, - 'max_height': 134217728, - 'min_y': 0, - 'max_y': 134217728, - 'max_zoom': 19, - 'tile_size': 256 + "min_x": 0, + "max_height": 134217728, + "min_y": 0, + "max_y": 134217728, + "max_zoom": 19, + "tile_size": 256, } continue if tileset_object is None: tileset_infos[tileset_uuid] = { - 'error': 'No such tileset with uid: {}'.format(tileset_uuid) + "error": "No such tileset with uid: {}".format(tileset_uuid) } continue if tileset_object.private and request.user != tileset_object.owner: # dataset is not public - tileset_infos[tileset_uuid] = {'error': "Forbidden"} + tileset_infos[tileset_uuid] = {"error": "Forbidden"} continue - if ( - tileset_object.filetype == 'hitile' or - tileset_object.filetype == 'hibed' - ): + if tileset_object.filetype == "hitile" or tileset_object.filetype == "hibed": tileset_info = hdft.get_tileset_info( - h5py.File(tileset_object.datafile.path, 'r')) + h5py.File(tileset_object.datafile.path, "r") + ) tileset_infos[tileset_uuid] = { - "min_pos": [int(tileset_info['min_pos'])], - "max_pos": [int(tileset_info['max_pos'])], - "max_width": 2 ** math.ceil( - math.log( - tileset_info['max_pos'] - tileset_info['min_pos'] - ) / math.log(2) + "min_pos": [int(tileset_info["min_pos"])], + "max_pos": [int(tileset_info["max_pos"])], + "max_width": 2 + ** math.ceil( + math.log(tileset_info["max_pos"] - tileset_info["min_pos"]) + / math.log(2) ), - "tile_size": int(tileset_info['tile_size']), - "max_zoom": int(tileset_info['max_zoom']) + "tile_size": int(tileset_info["tile_size"]), + "max_zoom": int(tileset_info["max_zoom"]), } - elif tileset_object.filetype == 'bigwig': + elif tileset_object.filetype == "bigwig": chromsizes = tgt.get_chromsizes(tileset_object) - tsinfo = hgbi.tileset_info( - tileset_object.datafile.path, - chromsizes - ) - #print('tsinfo:', tsinfo) - if 'chromsizes' in tsinfo: - tsinfo['chromsizes'] = [(c, int(s)) for c,s in tsinfo['chromsizes']] + tsinfo = hgbi.tileset_info(tileset_object.datafile.path, chromsizes) + # print('tsinfo:', tsinfo) + if "chromsizes" in tsinfo: + tsinfo["chromsizes"] = [(c, int(s)) for c, s in tsinfo["chromsizes"]] tileset_infos[tileset_uuid] = tsinfo - elif tileset_object.filetype == 'bigbed': + elif tileset_object.filetype == "bigbed": chromsizes = tgt.get_chromsizes(tileset_object) - tsinfo = hgbi.tileset_info( - tileset_object.datafile.path, - chromsizes - ) - #print('tsinfo:', tsinfo) - if 'chromsizes' in tsinfo: - tsinfo['chromsizes'] = [(c, int(s)) for c,s in tsinfo['chromsizes']] + tsinfo = hgbi.tileset_info(tileset_object.datafile.path, chromsizes) + # print('tsinfo:', tsinfo) + if "chromsizes" in tsinfo: + tsinfo["chromsizes"] = [(c, int(s)) for c, s in tsinfo["chromsizes"]] tileset_infos[tileset_uuid] = tsinfo - elif tileset_object.filetype == 'multivec': + elif tileset_object.filetype == "multivec": tileset_infos[tileset_uuid] = hgmu.tileset_info( - tileset_object.datafile.path) + tileset_object.datafile.path + ) elif tileset_object.filetype == "elastic_search": - response = urllib.urlopen( - tileset_object.datafile + "/tileset_info") + response = urllib.urlopen(tileset_object.datafile + "/tileset_info") tileset_infos[tileset_uuid] = json.loads(response.read()) - elif tileset_object.filetype == 'beddb': + elif tileset_object.filetype == "beddb": tileset_infos[tileset_uuid] = cdt.get_tileset_info( tileset_object.datafile.path ) - elif tileset_object.filetype == 'bed2ddb': + elif tileset_object.filetype == "bed2ddb": tileset_infos[tileset_uuid] = cdt.get_2d_tileset_info( tileset_object.datafile.path ) - elif tileset_object.filetype == 'cooler': + elif tileset_object.filetype == "cooler": tileset_infos[tileset_uuid] = hgco.tileset_info( - tileset_object.datafile.path + tileset_object.datafile.path ) - elif tileset_object.filetype == 'time-interval-json': + elif tileset_object.filetype == "time-interval-json": tileset_infos[tileset_uuid] = hgti.tileset_info( - tileset_object.datafile.path + tileset_object.datafile.path ) elif ( - tileset_object.filetype == '2dannodb' or - tileset_object.filetype == 'imtiles' + tileset_object.filetype == "2dannodb" + or tileset_object.filetype == "imtiles" ): tileset_infos[tileset_uuid] = hgim.get_tileset_info( tileset_object.datafile.path ) - elif tileset_object.filetype == 'geodb': + elif tileset_object.filetype == "geodb": tileset_infos[tileset_uuid] = hggo.tileset_info( tileset_object.datafile.path ) - elif tileset_object.filetype == 'bam': - tileset_infos[tileset_uuid] = ctb.tileset_info( - tileset_object.datafile.path - ) - tileset_infos[tileset_uuid]['max_tile_width'] = hss.MAX_BAM_TILE_WIDTH + elif tileset_object.filetype == "bam": + tileset_infos[tileset_uuid] = ctb.tileset_info(tileset_object.datafile.path) + tileset_infos[tileset_uuid]["max_tile_width"] = hss.MAX_BAM_TILE_WIDTH else: # Unknown filetype tileset_infos[tileset_uuid] = { - 'error': 'Unknown filetype ' + tileset_object.filetype + "error": "Unknown filetype " + tileset_object.filetype } - tileset_infos[tileset_uuid]['name'] = tileset_object.name - tileset_infos[tileset_uuid]['datatype'] = tileset_object.datatype - tileset_infos[tileset_uuid]['coordSystem'] = tileset_object.coordSystem - tileset_infos[tileset_uuid]['coordSystem2'] =\ - tileset_object.coordSystem2 + tileset_infos[tileset_uuid]["name"] = tileset_object.name + tileset_infos[tileset_uuid]["datatype"] = tileset_object.datatype + tileset_infos[tileset_uuid]["coordSystem"] = tileset_object.coordSystem + tileset_infos[tileset_uuid]["coordSystem2"] = tileset_object.coordSystem2 return JsonResponse(tileset_infos) -@api_view(['POST']) +@api_view(["POST"]) @authentication_classes((CsrfExemptSessionAuthentication, BasicAuthentication)) def link_tile(request): - ''' + """ A file has been uploaded to S3. Finish the upload here by adding the file to the database. @@ -671,40 +647,55 @@ def link_tile(request): Returns: JsonResponse: A response containing the uuid of the newly added tileset - ''' - body = json.loads(request.body.decode('utf8')) + """ + body = json.loads(request.body.decode("utf8")) media_base_path = op.realpath(hss.MEDIA_ROOT) - abs_filepath = op.realpath(op.join(media_base_path, body['filepath'])) + abs_filepath = op.realpath(op.join(media_base_path, body["filepath"])) if abs_filepath.find(media_base_path) != 0: # check ot make sure that the filename is contained in the AWS_BUCKET_MOUNT # e.g. that somebody isn't surreptitiously trying to pass in ('../../file') - return JsonResponse({'error': "Provided path ({}) not in the data path".format(body['filepath'])}, status=422) + return JsonResponse( + { + "error": "Provided path ({}) not in the data path".format( + body["filepath"] + ) + }, + status=422, + ) else: if not op.exists(abs_filepath): - return JsonResponse({'error': "Specified file ({}) does not exist".format(body['filepath'])}, status=400) + return JsonResponse( + { + "error": "Specified file ({}) does not exist".format( + body["filepath"] + ) + }, + status=400, + ) - diff_path = abs_filepath[len(media_base_path)+1:] # +1 for the slash + diff_path = abs_filepath[len(media_base_path) + 1 :] # +1 for the slash tile_data = body.copy() - tile_data.pop('filepath') + tile_data.pop("filepath") # print("user:", request.user) try: obj = tm.Tileset.objects.create( datafile=diff_path, - name=op.basename(body['filepath']), + name=op.basename(body["filepath"]), owner=request.user, - **tile_data) + **tile_data + ) except Exception as e: - return JsonResponse({'error': str(e)}, status=422) + return JsonResponse({"error": str(e)}, status=422) + return JsonResponse({"uuid": str(obj.uuid)}, status=201) - return JsonResponse({'uuid': str(obj.uuid)}, status=201) -@api_view(['POST']) +@api_view(["POST"]) def register_url(request): - ''' + """ Register a url to use as a tileset and register it with the database. Parameters: request: The HTTP request associate with this post action @@ -717,20 +708,20 @@ def register_url(request): coordSystem2: Returns: HttpResponse code for the request, 200 if the action is successful - ''' - body = json.loads(request.body.decode('utf8')) + """ + body = json.loads(request.body.decode("utf8")) - url = body.get('fileurl', '') + url = body.get("fileurl", "") if not url: - return JsonResponse({ 'error': 'No url provided in the fileurl field'}) + return JsonResponse({"error": "No url provided in the fileurl field"}) index_url = None - filetype = body.get('filetype', None) - if filetype == 'bam': - index_url = body.get('indexurl', None) + filetype = body.get("filetype", None) + if filetype == "bam": + index_url = body.get("indexurl", None) if not index_url: - return JsonResponse({ 'error': 'bam filetype requires an indexurl field'}) + return JsonResponse({"error": "bam filetype requires an indexurl field"}) """ # validate the url to ensure we didn't get garbage @@ -747,25 +738,25 @@ def register_url(request): # ingest the file by calling the ingest_tileset command new_obj = ingest_tileset_to_db( filename=url, - datatype=body.get('datatype', None), - filetype=body.get('filetype', None), - coordSystem=body.get('coordSystem', ''), - coordSystem2=body.get('coordSystem2', ''), - project_name=body.get('project_name', ''), - uid=body.get('uid', None), - name=body.get('name', None), + datatype=body.get("datatype", None), + filetype=body.get("filetype", None), + coordSystem=body.get("coordSystem", ""), + coordSystem2=body.get("coordSystem2", ""), + project_name=body.get("project_name", ""), + uid=body.get("uid", None), + name=body.get("name", None), indexfile=index_url, temporary=True, - no_upload=True + no_upload=True, ) except Exception as e: - logger.error('Problem registering url: %s' % e) - return JsonResponse({'error': str(e)}, status=500) + logger.error("Problem registering url: %s" % e) + return JsonResponse({"error": str(e)}, status=500) - return JsonResponse({ 'uid': new_obj.uuid }, content_type="text/plain") + return JsonResponse({"uid": new_obj.uuid}, content_type="text/plain") -@method_decorator(gzip_page, name='dispatch') +@method_decorator(gzip_page, name="dispatch") class TilesetsViewSet(viewsets.ModelViewSet): """Tilesets""" @@ -777,50 +768,84 @@ class TilesetsViewSet(viewsets.ModelViewSet): else: permission_classes = (tsp.UserPermissionReadOnly,) - lookup_field = 'uuid' - parser_classes = (rfp.JSONParser, rfp.MultiPartParser,) + lookup_field = "uuid" + parser_classes = (rfp.JSONParser, rfp.MultiPartParser) def destroy(self, request, *args, **kwargs): - '''Delete a tileset instance and underlying media upload - ''' - uuid = self.kwargs['uuid'] + """Delete a tileset instance and underlying media upload + """ + uuid = self.kwargs["uuid"] if not uuid: - return JsonResponse({'error': 'The uuid parameter is undefined'}, status=400) + return JsonResponse( + {"error": "The uuid parameter is undefined"}, status=400 + ) try: instance = self.get_object() self.perform_destroy(instance) filename = instance.datafile.name filepath = op.join(hss.MEDIA_ROOT, filename) if not op.isfile(filepath): - return JsonResponse({'error': 'Unable to locate tileset media file for deletion: {}'.format(filepath)}, status=500) + return JsonResponse( + { + "error": "Unable to locate tileset media file for deletion: {}".format( + filepath + ) + }, + status=500, + ) os.remove(filepath) except dh.Http404: - return JsonResponse({'error': 'Unable to locate tileset instance for uuid: {}'.format(uuid)}, status=404) + return JsonResponse( + { + "error": "Unable to locate tileset instance for uuid: {}".format( + uuid + ) + }, + status=404, + ) except dbm.ProtectedError as dbpe: - return JsonResponse({'error': 'Unable to delete tileset instance: {}'.format(str(dbpe))}, status=500) + return JsonResponse( + {"error": "Unable to delete tileset instance: {}".format(str(dbpe))}, + status=500, + ) except OSError: - return JsonResponse({'error': 'Unable to delete tileset media file: {}'.format(filepath)}, status=500) + return JsonResponse( + {"error": "Unable to delete tileset media file: {}".format(filepath)}, + status=500, + ) return HttpResponse(status=204) def retrieve(self, request, *args, **kwargs): - '''Retrieve a serialized JSON object made from a subset of properties of a tileset instance - ''' - uuid = self.kwargs['uuid'] + """Retrieve a serialized JSON object made from a subset of properties of a tileset instance + """ + uuid = self.kwargs["uuid"] if not uuid: - return JsonResponse({'error': 'The uuid parameter is undefined'}, status=400) + return JsonResponse( + {"error": "The uuid parameter is undefined"}, status=400 + ) try: queryset = tm.Tileset.objects.all().filter(uuid=uuid) except dce.ObjectDoesNotExist as dne: - return JsonResponse({'error': 'Unable to retrieve tileset instance: {}'.format(str(dne))}, status=500) + return JsonResponse( + {"error": "Unable to retrieve tileset instance: {}".format(str(dne))}, + status=500, + ) serializer = tss.UserFacingTilesetSerializer(queryset, many=True) try: instance = serializer.data[0] except IndexError as ie: - return JsonResponse({'error': 'Unable to locate tileset instance for uuid: {}'.format(uuid)}, status=404) + return JsonResponse( + { + "error": "Unable to locate tileset instance for uuid: {}".format( + uuid + ) + }, + status=404, + ) return JsonResponse(instance) def list(self, request, *args, **kwargs): - '''List the available tilesets + """List the available tilesets Args: request (django.http.HTTPRequest): The HTTP request containing @@ -829,34 +854,32 @@ def list(self, request, *args, **kwargs): Returns: django.http.JsonResponse: A json file containing a 'count' as well as 'results' with each tileset as an entry - ''' + """ # only return tilesets which are accessible by this user if request.user.is_anonymous: user = gu.get_anonymous_user() else: user = request.user - queryset = self.queryset.filter( - dbm.Q(owner=user) | dbm.Q(private=False) - ) + queryset = self.queryset.filter(dbm.Q(owner=user) | dbm.Q(private=False)) - if 'ac' in request.GET: + if "ac" in request.GET: # Autocomplete fields - queryset = queryset.filter(name__contains=request.GET['ac']) - if 't' in request.GET: + queryset = queryset.filter(name__contains=request.GET["ac"]) + if "t" in request.GET: # Filter by filetype - queryset = queryset.filter(filetype=request.GET['t']) - if 'dt' in request.GET: + queryset = queryset.filter(filetype=request.GET["t"]) + if "dt" in request.GET: # Filter by datatype - queryset = queryset.filter(datatype__in=request.GET.getlist('dt')) + queryset = queryset.filter(datatype__in=request.GET.getlist("dt")) - if 'o' in request.GET: - if 'r' in request.GET: - queryset = queryset.order_by(dbmf.Lower(request.GET['o']).desc()) + if "o" in request.GET: + if "r" in request.GET: + queryset = queryset.order_by(dbmf.Lower(request.GET["o"]).desc()) else: - queryset = queryset.order_by(dbmf.Lower(request.GET['o']).asc()) + queryset = queryset.order_by(dbmf.Lower(request.GET["o"]).asc()) - #ts_serializer = tss.UserFacingTilesetSerializer(queryset, many=True) + # ts_serializer = tss.UserFacingTilesetSerializer(queryset, many=True) page = self.paginate_queryset(queryset) if page is not None: serializer = self.get_serializer(page, many=True) @@ -872,7 +895,7 @@ def list(self, request, *args, **kwargs): """ def perform_create(self, serializer): - '''Add a new tileset + """Add a new tileset When adding a new dataset, we need to enforce permissions as well as other rules like the uniqueness of uuids. @@ -880,35 +903,32 @@ def perform_create(self, serializer): Args: serializer (tilsets.serializer.TilesetSerializer): The serializer to use to save the request. - ''' + """ - if 'uid' in self.request.data: + if "uid" in self.request.data: try: - self.queryset.get(uuid=self.request.data['uid']) + self.queryset.get(uuid=self.request.data["uid"]) # this uid already exists, return an error raise rfe.APIException("UID already exists") except tm.Tileset.DoesNotExist: - uid = self.request.data['uid'] + uid = self.request.data["uid"] else: uid = slugid.nice() - if 'filetype' not in self.request.data: - raise rfe.APIException('Missing filetype') + if "filetype" not in self.request.data: + raise rfe.APIException("Missing filetype") - datafile_name = self.request.data.get('datafile').name + datafile_name = self.request.data.get("datafile").name - if 'name' in self.request.data: - name = self.request.data['name'] + if "name" in self.request.data: + name = self.request.data["name"] else: name = op.split(datafile_name)[1] if self.request.user.is_anonymous: # can't create a private dataset as an anonymous user serializer.save( - owner=gu.get_anonymous_user(), - private=False, - name=name, - uuid=uid + owner=gu.get_anonymous_user(), private=False, name=name, uuid=uid ) else: serializer.save(owner=self.request.user, name=name, uuid=uid) diff --git a/website/tests.py b/website/tests.py index 834ed22e..d0296f3f 100644 --- a/website/tests.py +++ b/website/tests.py @@ -13,43 +13,37 @@ import higlass_server.settings as hss + class SiteTests(dt.TestCase): def setUp(self): - self.user1 = dcam.User.objects.create_user( - username='user1', password='pass' - ) + self.user1 = dcam.User.objects.create_user(username="user1", password="pass") - upload_json_text = json.dumps({'hi': 'there'}) + upload_json_text = json.dumps({"hi": "there"}) - self.viewconf = tm.ViewConf.objects.create( - viewconf=upload_json_text, uuid='md') + self.viewconf = tm.ViewConf.objects.create(viewconf=upload_json_text, uuid="md") def test_link_url(self): - ret = self.client.get('/link/') - assert "No uuid specified" in ret.content.decode('utf8') + ret = self.client.get("/link/") + assert "No uuid specified" in ret.content.decode("utf8") - ret = self.client.get('/link/?d=x') + ret = self.client.get("/link/?d=x") assert ret.status_code == 404 - ret = self.client.get('/link/?d=md') - assert ret.content.decode('utf8').find('window.location') >= 0 + ret = self.client.get("/link/?d=md") + assert ret.content.decode("utf8").find("window.location") >= 0 - @mock.patch('website.views.screenshot', new=CoroutineMock()) + @mock.patch("website.views.screenshot", new=CoroutineMock()) def test_thumbnail(self): - uuid = 'some_fake_uid' + uuid = "some_fake_uid" output_file = Path(hss.THUMBNAILS_ROOT) / (uuid + ".png") if not output_file.exists(): output_file.touch() - ret = self.client.get( - f'/thumbnail/?d={uuid}' - ) + ret = self.client.get(f"/thumbnail/?d={uuid}") self.assertEqual(ret.status_code, 200) - ret = self.client.get( - f'/t/?d=..file' - ) + ret = self.client.get(f"/t/?d=..file") self.assertEqual(ret.status_code, 400) diff --git a/website/urls.py b/website/urls.py index 5684079e..97a2b05d 100644 --- a/website/urls.py +++ b/website/urls.py @@ -4,9 +4,9 @@ # The API URLs are now determined automatically by the router. # Additionally, we include the login URLs for the browsable API. urlpatterns = [ - #url(r'^schema', schema_view), - url(r'^link/$', views.link), - url(r'^l/$', views.link), - url(r'^thumbnail/$', views.thumbnail), - url(r'^t/$', views.thumbnail) + # url(r'^schema', schema_view), + url(r"^link/$", views.link), + url(r"^l/$", views.link), + url(r"^thumbnail/$", views.thumbnail), + url(r"^t/$", views.thumbnail), ] diff --git a/website/views.py b/website/views.py index 6906f4bc..874d0b94 100644 --- a/website/views.py +++ b/website/views.py @@ -12,13 +12,18 @@ import higlass_server.settings as hss from django.core.exceptions import ObjectDoesNotExist -from django.http import HttpRequest, HttpResponse, \ - HttpResponseNotFound, HttpResponseBadRequest +from django.http import ( + HttpRequest, + HttpResponse, + HttpResponseNotFound, + HttpResponseBadRequest, +) logger = logging.getLogger(__name__) + def link(request): - '''Generate a small page containing the metadata necessary for + """Generate a small page containing the metadata necessary for link unfurling by Slack or Twitter. The generated page will point to a screenshot of the rendered viewconf. The page will automatically redirect to the rendering so that if anybody clicks on this link @@ -30,24 +35,24 @@ def link(request): request: The incoming http request. Returns: A response containing an html page with metadata - ''' + """ # the uuid of the viewconf to render - uuid = request.GET.get('d') + uuid = request.GET.get("d") if not uuid: # if there's no uuid specified, return an empty page - return HttpResponseNotFound('

No uuid specified

') + return HttpResponseNotFound("

No uuid specified

") try: obj = tm.ViewConf.objects.get(uuid=uuid) except ObjectDoesNotExist: - return HttpResponseNotFound('

No such uuid

') + return HttpResponseNotFound("

No such uuid

") # the url for the thumnbail - thumb_url=f'{request.scheme}://{request.get_host()}/thumbnail/?d={uuid}' + thumb_url = f"{request.scheme}://{request.get_host()}/thumbnail/?d={uuid}" # the page to redirect to for interactive explorations - redirect_url=f'{request.scheme}://{request.get_host()}/app/?config={uuid}' + redirect_url = f"{request.scheme}://{request.get_host()}/app/?config={uuid}" # Simple html page. Not a template just for simplicity's sake. # If it becomes more complex, we can make it into a template. @@ -79,8 +84,9 @@ def link(request): return HttpResponse(html) + def thumbnail(request: HttpRequest): - '''Retrieve a thumbnail for the viewconf specified by the d= + """Retrieve a thumbnail for the viewconf specified by the d= parameter. Args: @@ -89,17 +95,17 @@ def thumbnail(request: HttpRequest): A response of either 404 if there's no uuid provided or an image containing a screenshot of the rendered viewconf with that uuid. - ''' - uuid = request.GET.get('d') + """ + uuid = request.GET.get("d") - base_url = f'{request.scheme}://localhost/app/' + base_url = f"{request.scheme}://localhost/app/" if not uuid: - return HttpResponseNotFound('

No uuid specified

') + return HttpResponseNotFound("

No uuid specified

") - if '.' in uuid or '/' in uuid: + if "." in uuid or "/" in uuid: # no funny business - logger.warning('uuid contains . or /: %s', uuid) + logger.warning("uuid contains . or /: %s", uuid) return HttpResponseBadRequest("uuid can't contain . or /") if not op.exists(hss.THUMBNAILS_ROOT): @@ -109,31 +115,23 @@ def thumbnail(request: HttpRequest): thumbnails_base = op.abspath(hss.THUMBNAILS_ROOT) if output_file.find(thumbnails_base) != 0: - logger.warning('Thumbnail file is not in thumbnail_base: %s uuid: %s', - output_file, uuid) - return HttpResponseBadRequest('Strange path') + logger.warning( + "Thumbnail file is not in thumbnail_base: %s uuid: %s", output_file, uuid + ) + return HttpResponseBadRequest("Strange path") if not op.exists(output_file): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - loop.run_until_complete( - screenshot( - base_url, - uuid, - output_file)) + loop.run_until_complete(screenshot(base_url, uuid, output_file)) loop.close() - with open(output_file, 'rb') as file: - return HttpResponse( - file.read(), - content_type="image/jpeg") + with open(output_file, "rb") as file: + return HttpResponse(file.read(), content_type="image/jpeg") + -async def screenshot( - base_url: str, - uuid: str, - output_file: str -): - '''Take a screenshot of a rendered viewconf. +async def screenshot(base_url: str, uuid: str, output_file: str): + """Take a screenshot of a rendered viewconf. Args: base_url: The url to use for rendering the viewconf @@ -142,18 +140,16 @@ async def screenshot( the thumbnail. Returns: Nothing, just stores the screenshot at the given location. - ''' + """ browser = await launch( headless=True, - args=['--no-sandbox'], + args=["--no-sandbox"], handleSIGINT=False, handleSIGTERM=False, - handleSIGHUP=False + handleSIGHUP=False, ) - url = f'{base_url}?config={uuid}' + url = f"{base_url}?config={uuid}" page = await browser.newPage() - await page.goto(url, { - 'waitUntil': 'networkidle0', - }) - await page.screenshot({'path': output_file}) + await page.goto(url, {"waitUntil": "networkidle0"}) + await page.screenshot({"path": output_file}) await browser.close()