O2vae integration

.github/workflows/docker.yaml

@@ -2,8 +2,8 @@ name: Publish Docker
 on:
   push:
     branches:
-      - main
-      - master
+    - main
+    - master
 #   pull_request: ~
 
 env:
@@ -14,37 +14,29 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/[email protected]
-        with:
-          fetch-depth: 2
-      - name: Log in to the Container registry
-        uses: docker/[email protected]
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+    - name: Checkout
+      uses: actions/[email protected]
+      with:
+        fetch-depth: 2
+    - name: Log in to the Container registry
+      if: ${{ !env.ACT }}
+      uses: docker/[email protected]
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/[email protected]
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+    - name: Extract metadata (tags, labels) for Docker
+      id: meta
+      uses: docker/[email protected]
+      with:
+        images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
-      - name: Build and push Docker image (version tag)
-        if: steps.check-version.outputs.current-version
-        uses: docker/[email protected]
-        with:
-          context: .
-          push: true
-          tags: ghcr.io/${{ github.repository }}:${{ steps.check-version.outputs.current-version }}
-          labels: ${{ steps.meta.outputs.labels }}
-
-      - name: Build and push Docker image (latest tag)
-        if: steps.check-version.outputs.current-version
-        uses: docker/[email protected]
-        with:
-          context: .
-          push: true
-          tags: ghcr.io/${{ github.repository }}:latest
-          labels: ${{ steps.meta.outputs.labels }}
+    - name: Build and push Docker image (version tag)
+      if: steps.check-version.outputs.current-version
+      uses: docker/[email protected]
+      with:
+        context: .
+        push: true
+        tags: ghcr.io/${{ github.repository }}:${{ steps.check-version.outputs.current-version }}
+        labels: ${{ steps.meta.outputs.labels }}

.github/workflows/test.yaml

@@ -1,36 +1,44 @@
-# https://github.com/marketplace/actions/install-poetry-action
-name: test
-
-on: [pull_request,push]
-
+name: conda
+on: [push]
 jobs:
-  test:
+  constructor:
+    name: conda build (${{ matrix.python-version }}, ${{ matrix.os }})
+    runs-on: ${{ matrix.os }}-latest
     defaults:
       run:
-        shell: bash -l {0}
+        shell: ${{ matrix.shell }}
     strategy:
-      fail-fast: false
       matrix:
+        # os: [ubuntu, windows, macos]
+        os: [ubuntu]
         python-version: ["3.9"]
-        os: [ubuntu-latest]
-        # os: [ubuntu-18.04, macos-latest, windows-latest]
-    runs-on: ${{ matrix.os }}
+        include:
+        - os: ubuntu
+          shell: bash -l {0}
+          # - os: windows
+          #   shell: cmd /C call {0}
+          # - os: macos
+          #   shell: bash -l {0}
     steps:
-      - name: Check out repository
-        uses: actions/checkout@v2
-      - uses: conda-incubator/setup-miniconda@v2
-        with:
-          auto-update-conda: true
-          use-mamba: true
-          environment-file: environment.yml
-          python-version: ${{ matrix.python-version }}
-      - name: poetry env
-        run: poetry env use python
-      - name: Poetry lock
-        run: poetry lock     
-      - name: Install library
-        run: poetry install --no-interaction
-      # - name: Run tests
-      #   run: |
-      #     source .venv/bin/activate
-      #     pytest tests/
+    - uses: actions/checkout@v2
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/free-disk-space@main
+      with:
+        tool-cache: false
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: true
+        docker-images: true
+        swap-storage: true
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        environment-file: environment.yml
+        miniforge-variant: Mambaforge
+        miniforge-version: latest
+        mamba-version: "*"
+        use-mamba: true
+        python-version: ${{ matrix.python-version }}
+    - name: Run tests
+      run: |
+        make test

Makefile

@@ -9,101 +9,3 @@ download.data:
 test:
 	pytest
 
-
-GOOGLE_APPLICATION_CREDENTIALS=$(shell pwd)/credentials.json
-BUCKET_NAME=idr-hipsci
-TRAINING_DIR=idr0034-kilpinen-hipsci
-PROJECT=prj-ext-dev-bia-binder-113155
-
-JOB_PREFIX=vae
-JOB_NAME=$(JOB_PREFIX)_$(shell date +%Y%m%d_%H%M%S)
-JOB_DIR=gs://${BUCKET_NAME}/${JOB_NAME}/models
-DATA_DIR=gs://${BUCKET_NAME}/${TRAINING_DIR}
-
-.EXPORT_ALL_VARIABLES:
-	GOOGLE_APPLICATION_CREDENTIALS
-	BUCKET_NAME
-	TRAINING_DIR
-	JOB_PREFIX
-	JOB_NAME
-	JOB_DIR
-
-
-# MY_VAR := $(shell echo whatever)
-
-# test:
-# 	@echo MY_VAR IS $(MY_VAR)
-
-test:
-	@echo $$GOOGLE_APPLICATION_CREDENTIALS $$BUCKET_NAME $$TRAINING_DIR
-
-all: get_data_list build
-
-build:
-	conda activate torch
-	python idr_get_data.py
-
-get_data_list:
-	ls /nfs/bioimage/drop/idr*/**/*.tiff > file_list.txt
-	ls -u /nfs/bioimage/drop/idr*/**/*.tiff > file_list.txt
-
-run.on.cloud:
-	python idr_get_data_s3.py
-
-run.on.cloud.snake:
-	snakemake --use-conda --cores all \
-		--verbose --google-lifesciences \
-		--default-remote-prefix idr-hipsci \
-		--google-lifesciences-region eu-west2
-
-run.snake:
-	snakemake  --cores all -F --use-conda --verbose
-
-get.env.file:
-	conda env export --from-history -f environment.yml -n torch
-
-on.gcp:
-	gcloud ai-platform jobs submit training ${JOB_NAME} \
-	--region=europe-west2 \
-	--master-image-uri=gcr.io/cloud-ml-public/training/pytorch-gpu.1-9 \
-	--scale-tier=CUSTOM \
-	--master-machine-type=n1-standard-8 \
-	--master-accelerator=type=nvidia-tesla-t4,count=1 \
-	--job-dir=${JOB_DIR} \
-	--package-path=./trainer \
-	--module-name=trainer.train \
-	--stream-logs \
-	-- \
-	--num-epochs=10 \
-	--batch-size=100 \
-	--learning-rate=0.001 \
-	--gpus=1
-
-
-on.gcp.big:
-	gcloud ai-platform jobs submit training ${JOB_NAME} \
-	--region=europe-west2 \
-	--master-image-uri=gcr.io/cloud-ml-public/training/pytorch-gpu.1-9 \
-	--config=config.yaml \
-	--job-dir=${JOB_DIR} \
-	--package-path=./trainer \
-	--module-name=trainer.train \
-	--stream-logs \
-	-- \
-	--num-epochs=10 \
-	--batch-size=100 \
-	--learning-rate=0.001 \
-	--gpus=2 \
-	--accelerator='ddp'\
-	--num_nodes=3
-
-tensorboard:
-	tensorboard --logdir=gs://$(BUCKET_NAME)/${JOB_NAME}
-download.data:
-	kaggle competitions download -c data-science-bowl-2018
-
-test:
-	pytest
-
-download.idr:
-	rsync -avR --progress ctr26@noah-login:/nfs/bioimage/drop/idr0093-mueller-perturbation/ data/idr

README.md

@@ -63,6 +63,13 @@ This utility makes it simple to fetch the necessary datasets:
 ```bash
 make download.data
 ```
+If you don't have a Kaggle account you must create one and then follow the next steps:
+1. Install the Kaggle API package so you can download the data from the Makefile you have all the information in their [Github repository](https://github.com/Kaggle/kaggle-api).
+2. To use the Kaggle API you need also to create an API token.
+   You can found how to do it in their [documentation](https://github.com/Kaggle/kaggle-api#api-credentials)
+4. After that you will need to add your user and key in a file called `kaggle.json` in this location in your home directory `chmod 600 ~/.kaggle/kaggle.json`
+5. Don't forget to accept the conditions for the "2018 Data Science Bowl" on the Kaggle website.
+   Otherwise you would not be able to pull this data from the command line. 
 
 ### 4. Developer Installation:
 
@@ -88,4 +95,4 @@ bioimage_embed is licensed under the MIT License. Please refer to the [LICENSE](
 
 ---
 
-Happy Embedding! 🧬🔬
+Happy Embedding! 🧬🔬

bioimage_embed/augmentations.py

@@ -1,40 +1,6 @@
 import albumentations as A
 import cv2
 
-DEFAULT_AUGMENTATION = A.Compose(
-    [
-        # Flip the images horizontally or vertically with a 50% chance
-        A.OneOf(
-            [
-                A.HorizontalFlip(p=0.5),
-                A.VerticalFlip(p=0.5),
-            ],
-            p=0.5,
-        ),
-        # Rotate the images by a random angle within a specified range
-        A.Rotate(limit=45, p=0.5),
-        # Randomly scale the image intensity to adjust brightness and contrast
-        A.RandomGamma(gamma_limit=(80, 120), p=0.5),
-        # Apply random elastic transformations to the images
-        A.ElasticTransform(
-            alpha=1,
-            sigma=50,
-            alpha_affine=50,
-            p=0.5,
-        ),
-        # Shift the image channels along the intensity axis
-        A.ChannelShuffle(p=0.5),
-        # Add a small amount of noise to the images
-        A.GaussNoise(var_limit=(10.0, 50.0), p=0.5),
-        # Crop a random part of the image and resize it back to the original size
-        A.RandomResizedCrop(
-            height=512, width=512, scale=(0.9, 1.0), ratio=(0.9, 1.1), p=0.5
-        ),
-        # Adjust image intensity with a specified range for individual channels
-        A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.5),
-    ]
-)
-
 from typing import Any
 
 import albumentations
@@ -43,6 +9,39 @@
 from omegaconf import DictConfig
 from PIL import Image
 
+DEFAULT_AUGMENTATION_LIST = [
+    # Flip the images horizontally or vertically with a 50% chance
+    A.OneOf(
+        [
+            A.HorizontalFlip(p=0.5),
+            A.VerticalFlip(p=0.5),
+        ],
+        p=0.5,
+    ),
+    # Rotate the images by a random angle within a specified range
+    A.Rotate(limit=45, p=0.5),
+    # Randomly scale the image intensity to adjust brightness and contrast
+    A.RandomGamma(gamma_limit=(80, 120), p=0.5),
+    # Apply random elastic transformations to the images
+    A.ElasticTransform(
+        alpha=1,
+        sigma=50,
+        alpha_affine=50,
+        p=0.5,
+    ),
+    # Shift the image channels along the intensity axis
+    A.ChannelShuffle(p=0.5),
+    # Add a small amount of noise to the images
+    A.GaussNoise(var_limit=(10.0, 50.0), p=0.5),
+    # Crop a random part of the image and resize it back to the original size
+    A.RandomResizedCrop(
+        height=512, width=512, scale=(0.9, 1.0), ratio=(0.9, 1.1), p=0.5
+    ),
+    # Adjust image intensity with a specified range for individual channels
+    A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.5),
+]
+
+DEFAULT_AUGMENTATION = A.Compose(DEFAULT_AUGMENTATION_LIST)
 
 class TransformsWrapper:
     def __init__(self, transforms_cfg: DictConfig) -> None:
@@ -81,9 +80,7 @@ def __init__(self, transforms_cfg: DictConfig) -> None:
                 _convert_="object",
             )
             valid_test_predict_aug.append(aug)
-        self.valid_test_predict_aug = albumentations.Compose(
-            valid_test_predict_aug
-        )
+        self.valid_test_predict_aug = albumentations.Compose(valid_test_predict_aug)
 
     def set_mode(self, mode: str) -> None:
         """Set `__call__` mode.
@@ -111,4 +108,4 @@ def __call__(self, image: Any, **kwargs: Any) -> Any:
             image = np.asarray(image)
         if self.mode == "train":
             return self.train_aug(image=image, **kwargs)
-        return self.valid_test_predict_aug(image=image, **kwargs)
+        return self.valid_test_predict_aug(image=image, **kwargs)

bioimage_embed/cli.py

@@ -0,0 +1,12 @@
+from .hydra import train, infer
+from typer import Typer
+
+app = Typer()
+app.command()(train)
+app.command()(infer)
+
+def main():
+    app()
+
+if __name__ == "__main__":
+    main()