testing adding data; all functionality working incl adding binding/ex…

…pr by providing regulator_locus_tag etc

.devcontainer/devcontainer.json

@@ -68,7 +68,8 @@
                 "batisteo.vscode-django",
                 "GitHub.copilot",
                 "ms-azuretools.vscode-docker",
-                "github.vscode-github-actions"
+                "github.vscode-github-actions",
+                "mechatroner.rainbow-csv"
             ]
         }
     },

.gitignore

@@ -3,6 +3,8 @@
 __pycache__/
 *.py[cod]
 *$py.class
+.ipynb_checkpoints
+*.ipynb
 
 # C extensions
 *.so

config/settings/base.py

@@ -348,5 +348,5 @@
 )
 CALLINGCARDS_PROMOTER_SIG_FORMAT = env(
     "CALLING_CARD_PROMOTER_SIG_FORMAT",
-    default="callingcards_promoter_sig",
+    default="cc_promoter_sig",
 )

docs/index.rst

@@ -12,6 +12,7 @@ Welcome to YeastRegulatoryDB's documentation!
 
    howto
    users
+   api/yeastregulatorydb.regulatory_data.models
 
 
 

requirements/base.txt

@@ -28,3 +28,6 @@ django-filter==23.5  # https://github.com/carltongibson/django-filter
 # Third Party Software
 # ------------------------------------------------------------------------------
 callingcardstools==1.4.1 # https://github.com/cmatkhan/callingcardstools
+# for interactive python shell
+ipython==8.18.1 # https://ipython.org
+notebook==7.0.6 # https://jupyter.org/

yeastregulatorydb/conftest.py

@@ -66,9 +66,8 @@ def callingcardsbackground(db) -> CallingCardsBackground:
     return CallingCardsBackgroundFactory()
 
 
-@pytest.fixture
-def chrmap(db, user: User) -> QuerySet:
-    data = [
+def chrmap_data() -> dict:
+    return [
         {
             "id": 1,
             "refseq": "NC_001133.9",
@@ -130,6 +129,11 @@ def chrmap(db, user: User) -> QuerySet:
             "type": "plasmid",
         },
     ]
+
+
+@pytest.fixture
+def chrmap(db, user: User) -> QuerySet:
+    data = chrmap_data()
     for record in data:
         record["uploader"] = user
         record["modifier"] = user
@@ -152,10 +156,9 @@ def datasource(db) -> DataSource:
     return DataSourceFactory()
 
 
-@pytest.fixture
-def fileformat(db) -> QuerySet:
+def fileformat_data() -> dict:
     # harb, hu both csvs
-    format_dict = {
+    return {
         "array": ({"gene_id": "int", "effect": "float", "pval": "float"}, ",", "effect", 0.0, "pval", 1.0, "none"),
         "qbed": (
             {"chr": "str", "start": "int", "end": "int", "depth": "int", "strand": "str"},
@@ -279,6 +282,11 @@ def fileformat(db) -> QuerySet:
             "feature",
         ),
     }
+
+
+@pytest.fixture
+def fileformat(db) -> QuerySet:
+    format_dict = fileformat_data()
     for key, value in format_dict.items():
         fields, separator, effect, effect_thres, pval, pval_thres, feature_identifier_col = value
         FileFormatFactory.create(
@@ -382,7 +390,7 @@ def genomicfeature(db) -> GenomicFeature:
 
 @pytest.fixture
 def genomicfeature_chr1_genes(db, chrmap: QuerySet, user: User) -> QuerySet:
-    chr1_genes_df = pd.read_csv(os.path.join(TEST_DATA_ROOT, "chr1_genes.csv.gz"), compression="gzip")
+    chr1_genes_df = pd.read_csv(os.path.join(TEST_DATA_ROOT, "genome/chr1_genes.csv.gz"), compression="gzip")
     chr1_genes_dict = chr1_genes_df.to_dict(orient="records")
     for record in chr1_genes_dict:
         record["chr"] = chrmap.first()
@@ -406,7 +414,7 @@ def promotersetsig(db) -> PromoterSetSig:
 @pytest.fixture
 def regulator(db) -> Regulator:
     hap5_genomic_feature = GenomicFeatureFactory(locus_tag="YOR358W", symbol="HAP5")
-    return RegulatorFactory(id=1, regulator=hap5_genomic_feature)
+    return RegulatorFactory(id=1, genomicfeature=hap5_genomic_feature)
 
 
 @pytest.fixture
@@ -415,3 +423,25 @@ def rankresponse(db, regulator: Regulator) -> QuerySet:
     expression = ExpressionFactory(regulator=regulator)
     promotersetsig = PromoterSetSigFactory(binding=binding)
     return RankResponseFactory(promotersetsig=promotersetsig, expression=expression)
+
+
+@pytest.fixture
+def test_data_dict() -> dict:
+    root = os.path.join(os.path.dirname(__file__), "regulatory_data/tests/test_data")
+    file_dict = {}
+    for dirpath, dirnames, filenames in os.walk(root):
+        for filename in filenames:
+            path = os.path.join(dirpath, filename)
+            relative_dir = os.path.relpath(dirpath, root)
+            dir_components = relative_dir.split(os.sep)
+
+            # Create nested dictionaries for each directory component
+            current_dict = file_dict
+            for component in dir_components:
+                if component not in current_dict:
+                    current_dict[component] = {}
+                current_dict = current_dict[component]
+
+            # Add the file to the innermost dictionary
+            current_dict.setdefault("files", []).append(path)
+    return file_dict

yeastregulatorydb/regulatory_data/admin.py

@@ -1,4 +1,3 @@
-# pylint: disable=unused-import
 from django.contrib import admin
 
 # Register your models here.
@@ -14,18 +13,20 @@
     GenomicFeature,
     PromoterSet,
     PromoterSetSig,
+    RankResponse,
     Regulator,
 )
 
-admin.register(Binding)
-admin.register(BindingManualQC)
-admin.register(DataSource)
-admin.register(CallingCardsBackground)
-admin.register(ChrMap)
-admin.register(Expression)
-admin.register(ExpressionManualQC)
-admin.register(FileFormat)
-admin.register(GenomicFeature)
-admin.register(PromoterSet)
-admin.register(PromoterSetSig)
-admin.register(Regulator)
+admin.site.register(Binding)
+admin.site.register(BindingManualQC)
+admin.site.register(DataSource)
+admin.site.register(CallingCardsBackground)
+admin.site.register(ChrMap)
+admin.site.register(Expression)
+admin.site.register(ExpressionManualQC)
+admin.site.register(FileFormat)
+admin.site.register(GenomicFeature)
+admin.site.register(PromoterSet)
+admin.site.register(PromoterSetSig)
+admin.site.register(RankResponse)
+admin.site.register(Regulator)

yeastregulatorydb/regulatory_data/api/filters/BindingFilter.py

@@ -8,8 +8,10 @@ class BindingFilter(django_filters.FilterSet):
     id = django_filters.NumberFilter()
     pk = django_filters.NumberFilter()
     regulator = django_filters.NumberFilter()
-    regulator_locus_tag = django_filters.CharFilter(field_name="regulator__regulator__locus_tag", lookup_expr="iexact")
-    regulator_symbol = django_filters.CharFilter(field_name="regulator__regulator__symbol", lookup_expr="iexact")
+    regulator_locus_tag = django_filters.CharFilter(
+        field_name="regulator__genomicfeature__locus_tag", lookup_expr="iexact"
+    )
+    regulator_symbol = django_filters.CharFilter(field_name="regulator__genomicfeature__symbol", lookup_expr="iexact")
     batch = django_filters.CharFilter(lookup_expr="iexact")
     # pylint: enable=R0801
     replicate = django_filters.NumberFilter()

yeastregulatorydb/regulatory_data/api/filters/BindingManualQCFilter.py

@@ -7,13 +7,16 @@ class BindingManualQCFilter(django_filters.FilterSet):
     id = django_filters.NumberFilter()
     pk = django_filters.NumberFilter()
     binding = django_filters.NumberFilter()
-    rank_response_pass = django_filters.BooleanFilter()
-    best_response_pass = django_filters.BooleanFilter()
+    best_datatype = django_filters.BooleanFilter()
     data_usable = django_filters.BooleanFilter()
     passing_replicate = django_filters.BooleanFilter()
     regulator = django_filters.NumberFilter(field_name="binding__regulator")
-    regulator_locus_tag = django_filters.CharFilter(field_name="binding__regulator__locus_tag", lookup_expr="iexact")
-    regulator_symbol = django_filters.CharFilter(field_name="binding__regulator__symbol", lookup_expr="iexact")
+    regulator_locus_tag = django_filters.CharFilter(
+        field_name="binding__regulator__genomicfeature__locus_tag", lookup_expr="iexact"
+    )
+    regulator_symbol = django_filters.CharFilter(
+        field_name="binding__regulator__genomicfeature__symbol", lookup_expr="iexact"
+    )
     batch = django_filters.CharFilter(field_name="binding__batch", lookup_expr="iexact")
     source = django_filters.CharFilter(field_name="binding__source", lookup_expr="iexact")
 
@@ -23,8 +26,7 @@ class Meta:
             "id",
             "pk",
             "binding",
-            "rank_response_pass",
-            "best_response_pass",
+            "best_datatype",
             "data_usable",
             "passing_replicate",
             "regulator",

yeastregulatorydb/regulatory_data/api/filters/ExpressionFilter.py

@@ -8,8 +8,10 @@ class ExpressionFilter(django_filters.FilterSet):
     id = django_filters.NumberFilter()
     pk = django_filters.NumberFilter()
     regulator = django_filters.NumberFilter()
-    regulator_locus_tag = django_filters.CharFilter(field_name="regulator__regulator__locus_tag", lookup_expr="iexact")
-    regulator_symbol = django_filters.CharFilter(field_name="regulator__regulator__symbol", lookup_expr="iexact")
+    regulator_locus_tag = django_filters.CharFilter(
+        field_name="regulator__genomicfeature__locus_tag", lookup_expr="iexact"
+    )
+    regulator_symbol = django_filters.CharFilter(field_name="regulator__genomicfeature__symbol", lookup_expr="iexact")
     batch = django_filters.CharFilter(field_name="batch", lookup_expr="iexact")
     # pylint: enable=R0801
     replicate = django_filters.NumberFilter()

yeastregulatorydb/regulatory_data/api/filters/ExpressionManualQCFilter.py

@@ -9,10 +9,10 @@ class ExpressionManualQCFilter(django_filters.FilterSet):
     expression = django_filters.NumberFilter()
     strain_verified = django_filters.ChoiceFilter(choices=[("yes", "yes"), ("no", "no"), ("unverified", "unverified")])
     regulator_locus_tag = django_filters.CharFilter(
-        field_name="expression__regulator__regulator__locus_tag", lookup_expr="iexact"
+        field_name="expression__regulator__genomicfeature__locus_tag", lookup_expr="iexact"
     )
     regulator_symbol = django_filters.CharFilter(
-        field_name="expression__regulator__regulator__symbol", lookup_expr="iexact"
+        field_name="expression__regulator__genomicfeature__symbol", lookup_expr="iexact"
     )
     batch = django_filters.CharFilter(field_name="expression__batch", lookup_expr="iexact")
     replicate = django_filters.NumberFilter(field_name="expression__replicate")

yeastregulatorydb/regulatory_data/api/filters/PromoterSetSigFilter.py

@@ -12,10 +12,10 @@ class PromoterSetSigFilter(django_filters.FilterSet):
     background = django_filters.NumberFilter()
     background_name = django_filters.CharFilter(field_name="background_id__name", lookup_expr="iexact")
     regulator_locus_tag = django_filters.CharFilter(
-        field_name="binding__regulator__regulator__locus_tag", lookup_expr="iexact"
+        field_name="binding__regulator__genomicfeature__locus_tag", lookup_expr="iexact"
     )
     regulator_symbol = django_filters.CharFilter(
-        field_name="binding__regulator__regulator__symbol", lookup_expr="iexact"
+        field_name="binding__regulator__genomicfeature__symbol", lookup_expr="iexact"
     )
     batch = django_filters.CharFilter(field_name="binding__batch", lookup_expr="iexact")
     replicate = django_filters.NumberFilter(field_name="binding__replicate")

yeastregulatorydb/regulatory_data/api/filters/RankResponseFilter.py

@@ -13,10 +13,10 @@ class RankResponseFilter(django_filters.FilterSet):
     expression_id = django_filters.NumberFilter(field_name="expression__id")
     expression_source = django_filters.CharFilter(field_name="expression__source__name", lookup_expr="iexact")
     regulator_locus_tag = django_filters.CharFilter(
-        field_name="expression__regulator__regulator__locus_tag", lookup_expr="iexact"
+        field_name="expression__regulator__genomicfeature__locus_tag", lookup_expr="iexact"
     )
     regulator_symbol = django_filters.CharFilter(
-        field_name="expression__regulator__regulator__symbol", lookup_expr="iexact"
+        field_name="expression__regulator__genomicfeature__symbol", lookup_expr="iexact"
     )
     expression_effect_threshold = django_filters.NumberFilter()
     expression_pvalue_threshold = django_filters.NumberFilter()

yeastregulatorydb/regulatory_data/api/filters/RegulatorFilter.py

@@ -6,8 +6,8 @@
 class RegulatorFilter(django_filters.FilterSet):
     id = django_filters.NumberFilter(field_name="id")
     pk = django_filters.NumberFilter(field_name="pk")
-    regulator_locus_tag = django_filters.CharFilter(field_name="regulator__locus_tag", lookup_expr="iexact")
-    regulator_symbol = django_filters.CharFilter(field_name="regulator__symbol", lookup_expr="iexact")
+    regulator_locus_tag = django_filters.CharFilter(field_name="genomicfeature__locus_tag", lookup_expr="iexact")
+    regulator_symbol = django_filters.CharFilter(field_name="genomicfeature__symbol", lookup_expr="iexact")
     under_development = django_filters.BooleanFilter(field_name="under_development")
 
     class Meta:

yeastregulatorydb/regulatory_data/api/serializers/BindingSerializer.py

@@ -3,16 +3,27 @@
 from rest_framework import serializers
 
 from ...models import Binding
-from .mixins.CustomValidateMixin import CustomValidateMixin
-from .mixins.FileValidationMixin import FileValidationMixin
+from .mixins import CustomValidateMixin, FileValidationMixin, GetDataSourceMixin, GetOrCreateRegulatorMixin
 
 logger = logging.getLogger(__name__)
 
 
-class BindingSerializer(CustomValidateMixin, FileValidationMixin, serializers.ModelSerializer):
+class BindingSerializer(
+    GetDataSourceMixin,
+    GetOrCreateRegulatorMixin,
+    CustomValidateMixin,
+    FileValidationMixin,
+    serializers.ModelSerializer,
+):
     uploader = serializers.ReadOnlyField(source="uploader.username")
     modifier = serializers.CharField(source="uploader.username", required=False)
 
     class Meta:
         model = Binding
         fields = "__all__"
+
+    def to_representation(self, instance):
+        ret = super().to_representation(instance)
+        # Add the custom attribute to the serialized data
+        ret["promotersetsig_processing"] = getattr(instance, "promotersetsig_processing", False)
+        return ret

yeastregulatorydb/regulatory_data/api/serializers/BulkUploadSerializer.py

@@ -0,0 +1,6 @@
+from rest_framework import serializers
+
+
+class BulkUploadSerializer(serializers.Serializer):
+    csv_file = serializers.FileField()
+    tarred_dir = serializers.FileField()

yeastregulatorydb/regulatory_data/api/serializers/CallingCardsBackgroundSerializer.py

@@ -12,3 +12,9 @@ class CallingCardsBackgroundSerializer(CustomValidateMixin, FileValidationMixin,
     class Meta:
         model = CallingCardsBackground
         fields = "__all__"
+
+    def to_representation(self, instance):
+        ret = super().to_representation(instance)
+        # Add the custom attribute to the serialized data
+        ret["promotersetsig_processing"] = getattr(instance, "promotersetsig_processing", False)
+        return ret

yeastregulatorydb/regulatory_data/api/serializers/ExpressionSerializer.py

@@ -1,14 +1,25 @@
 from rest_framework import serializers
 
 from ...models.Expression import Expression
-from .mixins.CustomValidateMixin import CustomValidateMixin
-from .mixins.FileValidationMixin import FileValidationMixin
+from .mixins import CustomValidateMixin, FileValidationMixin, GetDataSourceMixin, GetOrCreateRegulatorMixin
 
 
-class ExpressionSerializer(CustomValidateMixin, FileValidationMixin, serializers.ModelSerializer):
+class ExpressionSerializer(
+    GetOrCreateRegulatorMixin,
+    GetDataSourceMixin,
+    CustomValidateMixin,
+    FileValidationMixin,
+    serializers.ModelSerializer,
+):
     uploader = serializers.ReadOnlyField(source="uploader.username")
     modifier = serializers.CharField(source="uploader.username", required=False)
 
     class Meta:
         model = Expression
         fields = "__all__"
+
+    def to_representation(self, instance):
+        ret = super().to_representation(instance)
+        # Add the custom attribute to the serialized data
+        ret["promotersetsig_processing"] = getattr(instance, "promotersetsig_processing", False)
+        return ret

yeastregulatorydb/regulatory_data/api/serializers/FileFormatSerializer.py

@@ -44,15 +44,27 @@ def validate_fields(self, value):
                 )
         return value
 
-    def to_representation(self, instance):
-        out = super().to_representation(instance)
-        fields_dict = out["fields"]
+    @property
+    def fields_as_types(self):
+        fields_dict = self.to_representation(self.instance)["fields"]
         for key, value in fields_dict.items():
             if value == "str":
                 fields_dict[key] = str
             elif value == "int":
                 fields_dict[key] = int
             elif value == "float":
                 fields_dict[key] = float
-        out["fields"] = fields_dict
-        return out
+        return fields_dict
+
+    # def to_representation(self, instance):
+    #     out = super().to_representation(instance)
+    #     fields_dict = out["fields"]
+    #     for key, value in fields_dict.items():
+    #         if value == "str":
+    #             fields_dict[key] = str
+    #         elif value == "int":
+    #             fields_dict[key] = int
+    #         elif value == "float":
+    #             fields_dict[key] = float
+    #     out["fields"] = fields_dict
+    #     return out

yeastregulatorydb/regulatory_data/api/serializers/PromoterSetSerializer.py

@@ -12,3 +12,9 @@ class PromoterSetSerializer(CustomValidateMixin, FileValidationMixin, serializer
     class Meta:
         model = PromoterSet
         fields = "__all__"
+
+    def to_representation(self, instance):
+        ret = super().to_representation(instance)
+        # Add the custom attribute to the serialized data
+        ret["promotersetsig_processing"] = getattr(instance, "promotersetsig_processing", False)
+        return ret