Re-enabled linting in build process

.github/workflows/lint.yml

@@ -2,10 +2,9 @@ name: Lint
 on:
   push:
     branches:
-      - master
+      - 'main'
   pull_request:
-    branches:
-      - '**'
+
 
 concurrency:
   cancel-in-progress: true
@@ -17,36 +16,37 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v3.1.4
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
-      - uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('deeplake/requirements/*.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
+          python-version: "3.10"
+          cache: pip
+          cache-dependency-path: deeplake/requirements/*.txt
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -r deeplake/requirements/common.txt
           pip install -r deeplake/requirements/tests.txt
           pip install -r deeplake/requirements/plugins.txt
+
       - name: Install deeplake
         run: pip install -e .
+
       - name: Check formatting with black
-        continue-on-error: true
+        if: always()
         run: |
           black --version
           black --check .
+
       - name: Lint docstrings with darglint
-        continue-on-error: true
+        if: always()
         run: |
           darglint --version
           darglint .
+
       - name: Check typing with mypy
-        continue-on-error: true
+        if: always()
         run: |
           mypy --version
           mypy .

deeplake/auto/structured/dataframe.py

@@ -7,7 +7,7 @@
 from deeplake.util.dataset import sanitize_tensor_name
 
 from collections import defaultdict
-from typing import DefaultDict, List, Union, Optional
+from typing import DefaultDict, List, Union, Optional, Dict
 from deeplake.core.sample import Sample
 from deeplake.core.linked_sample import LinkedSample
 import pathlib
@@ -53,7 +53,7 @@ def _initialize_params(self, column_params):
                 column_params[key] = {"name": sanitize_tensor_name(key)}
         self.column_params = column_params
 
-    def _get_most_frequent_image_extension(self, fn_iterator):
+    def _get_most_frequent_image_extension(self, fn_iterator: List[str]):
         # TODO: Make this generic and work for any htype that requires compression
 
         if len(fn_iterator) == 0:
@@ -77,7 +77,7 @@ def _get_most_frequent_image_extension(self, fn_iterator):
         )
         return most_frequent_image_extension
 
-    def _parse_tensor_params(self, key, inspect_limit=1000):
+    def _parse_tensor_params(self, key: str, inspect_limit: int = 1000):
         """Parse the tensor parameters for a column. Required parameters that are not specified will be inferred by inspecting up to 'inspect_limit' rows in the data."""
 
         tensor_params: Dict = self.column_params[key]

deeplake/core/chunk_engine.py

@@ -2,7 +2,7 @@
 from deeplake.client.log import logger
 import deeplake
 import numpy as np
-from tqdm import tqdm  # type: ignore
+from tqdm import tqdm
 from typing import (
     Any,
     Callable,
@@ -1740,6 +1740,7 @@
         if not buffer:
             return b""
         if self.is_sequence:
+            assert self.sequence_encoder is not None
             start_idx, end_idx = self.sequence_encoder[global_sample_index]
             end_idx -= 1
             start_idx, end_idx = map(
@@ -2211,7 +2212,7 @@
             for chunk in self.list_all_chunks()
         ]
 
-    def list_orphaned_chunks(self, storage):
+    def list_orphaned_chunks(self, storage: StorageProvider) -> List[str]:
         """Return paths for orphaned chunks (chunks what are not linked to the `current_version`)"""
 
         commit_id = self.commit_id
@@ -2243,6 +2244,7 @@
     ):
         if global_sample_index is None:
             if self.is_sequence:
+                assert self.sequence_encoder is not None
                 global_sample_index = self.sequence_encoder.num_samples - 1
             else:
                 global_sample_index = self.num_samples - 1
@@ -2264,6 +2266,7 @@
 
         self.commit_diff.pop(global_sample_index, sample_id)
         if self.is_sequence:
+            assert self.sequence_encoder is not None
             # pop in reverse order else indices get shifted
             for idx in reversed(range(*self.sequence_encoder[global_sample_index])):
                 self.pop_item(idx)
@@ -2431,6 +2434,7 @@
             return self.get_empty_sample()
         if index.subscriptable_at(0) and index.subscriptable_at(1):
             item_lengths = []
+            assert self.sequence_encoder is not None
             for i in index.values[0].indices(self._sequence_length):
                 item_length = index.length_at(
                     1, -int(np.subtract(*self.sequence_encoder[i]))
@@ -2475,6 +2479,7 @@
                     for j in y.indices(_item_length):
                         yield i * _item_length + j
 
+        assert self.sequence_encoder is not None
         idx0_gen.__len__ = (  # type: ignore
             (
                 lambda: sum(
@@ -2693,6 +2698,7 @@
                     sample_shapes = np.zeros((num_samples, sample_ndim), dtype=np.int32)
 
             if flatten:
+                assert self.sequence_encoder is not None
                 # fill sample shapes with sequence item shapes, no nesting
                 start, end = self.sequence_encoder[idx]
                 length = end - start
@@ -2993,6 +2999,7 @@
                     chunk_engine._transform_callback(vs, flat)
         except Exception:
             for k, num_samples in updated_tensors.items():
+                assert self._all_chunk_engines is not None
                 chunk_engine = self._all_chunk_engines[k]
                 num_samples_added = chunk_engine.tensor_length - num_samples
                 for _ in range(num_samples_added):
@@ -3009,6 +3016,8 @@
 
                 if flat_links:
                     seq_enc = self.sequence_encoder
+                    assert seq_enc is not None
+                    assert self._all_chunk_engines is not None
                     for link in flat_links:
                         link_chunk_engine = self._all_chunk_engines[link]
                         for idx in reversed(range(*seq_enc[index])):

deeplake/core/dataset/dataset.py

@@ -10,7 +10,7 @@
 import pathlib
 import numpy as np
 from time import time, sleep
-from tqdm import tqdm  # type: ignore
+from tqdm import tqdm
 
 import deeplake
 from deeplake.core.index.index import IndexEntry

deeplake/core/linked_chunk_engine.py

@@ -153,7 +153,9 @@ def get_video_sample(self, global_sample_index, index, decompress=True):
         except Exception as e:
             raise GetDataFromLinkError(path)
 
-    def get_full_tiled_sample(self, global_sample_index, fetch_chunks=False):
+    def get_full_tiled_sample(
+        self, global_sample_index: int, fetch_chunks: bool = False
+    ):
         tile_enc = self.tile_encoder
         shape = tile_enc.get_sample_shape(global_sample_index)
         tile_shape = tile_enc.get_tile_shape(global_sample_index)

deeplake/core/meta/encode/tile.py

@@ -6,7 +6,11 @@
 
 
 class TileEncoder(DeepLakeMemoryObject):
-    def __init__(self, entries=None, version=None):
+    def __init__(
+        self,
+        entries: Optional[Dict[int, Tuple[Tuple[int, ...], Tuple[int, ...]]]] = None,
+        version: Optional[str] = None,
+    ):
         self.is_dirty = False
         self.entries: Dict[int, Tuple[Tuple[int, ...], Tuple[int, ...]]] = entries or {}
         self.version = version or deeplake.__version__

deeplake/core/partial_sample.py

@@ -9,7 +9,7 @@ def __init__(
         self,
         sample_shape: Tuple[int, ...],
         tile_shape: Optional[Tuple[int, ...]] = None,
-        dtype: Union[str, np.dtype] = np.dtype("uint8"),
+        dtype: Optional[Union[str, np.dtype]] = np.dtype("uint8"),
     ):
         self.sample_shape = sample_shape
         self.tile_shape = tile_shape

deeplake/core/query/filter.py

@@ -288,9 +288,9 @@ def filter_inplace(
         vds_thread = _get_vds_thread(vds, vds_queue, num_samples)
         vds_thread.start()
     if progressbar:
-        from tqdm import tqdm  # type: ignore
+        from tqdm import tqdm
 
-        it = tqdm(it, total=num_samples)
+        it = tqdm(it, total=num_samples)  # type: ignore
 
     query_id = hash_inputs(dataset.path, dataset.pending_commit_id, query_text)
 

deeplake/core/tensor.py

@@ -1112,6 +1112,7 @@
 
             if flat_links:
                 seq_enc = self.chunk_engine.sequence_encoder
+                assert seq_enc is not None
                 for link in flat_links:
                     link_tensor = self.dataset[rev_tensor_names.get(link)]
                     for idx in reversed(range(*seq_enc[global_sample_index])):
@@ -1159,6 +1160,7 @@
         if self.is_sequence:
 
             def get_sample_shape(global_sample_index: int):
+                assert self.chunk_engine.sequence_encoder is not None
                 seq_pos = slice(
                     *self.chunk_engine.sequence_encoder[global_sample_index]
                 )
@@ -1179,6 +1181,7 @@
 
     def _get_sample_info_at_index(self, global_sample_index: int, sample_info_tensor):
         if self.is_sequence:
+            assert self.chunk_engine.sequence_encoder is not None
             return [
                 sample_info_tensor[i].data()
                 for i in range(*self.chunk_engine.sequence_encoder[global_sample_index])

deeplake/core/vectorstore/deeplake_vectorstore.py

@@ -295,7 +295,7 @@ def add(
 
     def search(
         self,
-        embedding_data: Union[str, List[str]] = None,
+        embedding_data: Union[str, List[str], None] = None,
         embedding_function: Optional[Callable] = None,
         embedding: Optional[Union[List[float], np.ndarray]] = None,
         k: int = 4,

deeplake/core/vectorstore/vector_search/dataset/dataset.py

@@ -393,7 +393,7 @@ def extend(
     embedding_function: List[Callable],
     embedding_data: List[Any],
     embedding_tensor: Union[str, List[str]],
-    processed_tensors: Dict[str, List[Any]],
+    processed_tensors: Dict[str, Union[List[Any], np.ndarray]],
     dataset: deeplake.core.dataset.Dataset,
 ):
     """
@@ -437,14 +437,14 @@ def extend(
                     if diff > 0:
                         time.sleep(diff)
             try:
-                embedded_data = np.vstack(embedded_data).astype(dtype=np.float32)
+                return_embedded_data = np.vstack(embedded_data).astype(dtype=np.float32)
             except ValueError:
                 raise IncorrectEmbeddingShapeError()
 
-            if len(embedded_data) == 0:
+            if len(return_embedded_data) == 0:
                 raise ValueError("embedding function returned empty list")
 
-            processed_tensors[tensor] = embedded_data
+            processed_tensors[tensor] = return_embedded_data
 
     dataset.extend(processed_tensors)
 

deeplake/core/vectorstore/vector_search/filter/filter.py

@@ -67,7 +67,7 @@ def attribute_based_filtering_tql(
                     val_str = (
                         f"'{filter[tensor]}'"
                         if isinstance(filter[tensor], str)
-                        or isinstance(filter[tensor], np._str)
+                        or isinstance(filter[tensor], np.str_)
                         else f"{filter[tensor]}"
                     )
                     tql_filter += f"{tensor} == {val_str} and "

deeplake/core/vectorstore/vector_search/indra/search_algorithm.py

@@ -68,7 +68,7 @@ def search(
     else:
         if not INDRA_INSTALLED:
             raise raise_indra_installation_error(
-                indra_import_error=False
+                indra_import_error=None
             )  # pragma: no cover
         return_data = {}
 

deeplake/core/version_control/dataset_diff.py

@@ -3,6 +3,7 @@
 from deeplake.util.keys import get_dataset_diff_key
 import typing
 from collections import OrderedDict
+import deeplake.core.dataset
 
 
 class DatasetDiff(DeepLakeMemoryObject):
@@ -118,7 +119,7 @@ def tensor_deleted(self, name):
             self.is_dirty = True
 
 
-def load_dataset_diff(dataset):
+def load_dataset_diff(dataset: "deeplake.core.dataset.Dataset"):
     storage: LRUCache = dataset.storage
     path = get_dataset_diff_key(dataset.version_state["commit_id"])
     try:

deeplake/enterprise/convert_to_libdeeplake.py

@@ -1,3 +1,5 @@
+from deeplake import Dataset
+
 from deeplake.core.storage.gcs import GCSProvider
 from deeplake.enterprise.util import raise_indra_installation_error  # type: ignore
 from deeplake.core.storage import S3Provider
@@ -143,7 +145,7 @@ def _get_indra_ds_from_s3_provider(
         )
 
 
-def dataset_to_libdeeplake(hub2_dataset):
+def dataset_to_libdeeplake(hub2_dataset: Dataset):
     """Convert a hub 2.x dataset object to a libdeeplake dataset object."""
     try_flushing(hub2_dataset)
     api = import_indra_api()
@@ -208,6 +210,8 @@ def dataset_to_libdeeplake(hub2_dataset):
         hub2_dataset.libdeeplake_dataset = libdeeplake_dataset
     else:
         libdeeplake_dataset = hub2_dataset.libdeeplake_dataset
+
+    assert libdeeplake_dataset is not None
     commit_id = hub2_dataset.pending_commit_id
     libdeeplake_dataset.checkout(commit_id)
     slice_ = hub2_dataset.index.values[0].value

deeplake/enterprise/test_pytorch.py

@@ -758,7 +758,7 @@ def test_batch_sampler_attribute(local_auth_ds):
 @pytest.mark.slow
 @pytest.mark.flaky
 def test_pil_decode_method(local_auth_ds):
-    from indra.pytorch.exceptions import CollateExceptionWrapper
+    from indra.pytorch.exceptions import CollateExceptionWrapper  # type: ignore
 
     with local_auth_ds as ds:
         ds.create_tensor("x", htype="image", sample_compression="jpeg")

deeplake/integrations/huggingface/huggingface.py

@@ -4,7 +4,7 @@
 import posixpath
 import deeplake
 from typing import Optional
-from tqdm import tqdm  # type: ignore
+from tqdm import tqdm
 from deeplake.util.bugout_reporter import feature_report_path, deeplake_reporter
 
 

deeplake/integrations/mmdet/mmdet_utils.py

@@ -22,7 +22,7 @@
 import json
 import mmcv  # type: ignore
 import math
-from tqdm import tqdm  # type: ignore
+from tqdm import tqdm
 
 
 def _isArrayLike(obj):

deeplake/integrations/pytorch/dataset.py

@@ -144,7 +144,7 @@ def __init__(
             streaming.list_blocks()
         )
 
-    def __iter__(self):
+    def __iter__(self: "TorchDataset"):
         worker_info = torch.utils.data.get_worker_info()
         schedule: Schedule = self.schedules[0]
 

deeplake/integrations/pytorch/shuffle_buffer.py

@@ -9,7 +9,7 @@
 
 from PIL import Image  # type: ignore
 from io import BytesIO
-from tqdm import tqdm  # type: ignore
+from tqdm import tqdm
 from deeplake.util.warnings import always_warn
 from deeplake.constants import MB
 import deeplake

deeplake/requirements/tests.txt

@@ -15,6 +15,7 @@ darglint
 typing_extensions>=3.10.0.0
 types-requests
 types-click
+types-tqdm
 boto3-stubs[essential]
 
 lz4