From 98d60eab6ca18d47ead2ec5dfa867025b435f5f3 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Fri, 1 Nov 2024 08:30:53 +0100 Subject: [PATCH 01/45] Added filtering , searching and ordering for the datasets --- backend/core/tasks.py | 3 ++- backend/core/views.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index fc9bc303..fb27baaa 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -84,8 +84,9 @@ def train_model( import hot_fair_utilities import ramp.utils import tensorflow as tf - from hot_fair_utilities import preprocess, train + from hot_fair_utilities import preprocess from hot_fair_utilities.training import run_feedback + from hot_fair_utilities.training.ramp import train from predictor import download_imagery, get_start_end_download_coords training_instance = get_object_or_404(Training, id=training_id) diff --git a/backend/core/views.py b/backend/core/views.py index 8bbd2d35..b44f85eb 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -89,7 +89,22 @@ class DatasetViewSet( permission_classes = [IsOsmAuthenticated] public_methods = ["GET"] queryset = Dataset.objects.all() + filter_backends = ( + DjangoFilterBackend, + filters.SearchFilter, + filters.OrderingFilter, + ) serializer_class = DatasetSerializer # connecting serializer + filterset_fields = { + "status": ["exact"], + "created_at": ["exact", "gt", "gte", "lt", "lte"], + "last_modified": ["exact", "gt", "gte", "lt", "lte"], + "user": ["exact"], + "id": ["exact"], + "source_imagery": ["exact"], + } + ordering_fields = ["created_at", "last_modified", "id", "status"] + search_fields = ["name", "id"] class TrainingSerializer( From b5a9dec6a89af6703b481ba243dff45d010bc6ec Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Fri, 1 Nov 2024 12:23:25 +0100 Subject: [PATCH 02/45] Add authenticated user allowed methods for the AOI Creation --- backend/core/views.py | 1 + backend/login/permissions.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/backend/core/views.py b/backend/core/views.py index b44f85eb..cbfc9008 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -319,6 +319,7 @@ class AOIViewSet(viewsets.ModelViewSet): authentication_classes = [OsmAuthentication] permission_classes = [IsOsmAuthenticated] public_methods = ["GET"] + authenticated_user_allowed_methods = ["POST", "DELETE"] queryset = AOI.objects.all() serializer_class = AOISerializer # connecting serializer filter_backends = [DjangoFilterBackend] diff --git a/backend/login/permissions.py b/backend/login/permissions.py index b40c1390..15cf8ade 100644 --- a/backend/login/permissions.py +++ b/backend/login/permissions.py @@ -12,10 +12,8 @@ def has_permission(self, request, view): return True if request.user and request.user.is_authenticated: - # Global access - if request.user.is_staff or request.user.is_superuser: - return True - + # if request.user.is_staff or request.user.is_superuser: + # return True return True return False @@ -34,9 +32,12 @@ def has_object_permission(self, request, view, obj): if obj.user == request.user: return True else: - if request.method == "POST": - # if object doesn't have user in it then he has permission to access the object , considered as common object - return True + if request.user and request.user.is_authenticated: + authenticated_user_allowed_methods = getattr( + view, "authenticated_user_allowed_methods", [] + ) + if request.method in authenticated_user_allowed_methods: + return True return False From 999445bca01336fabc63fcab2650c3f24c5c4220 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Fri, 1 Nov 2024 12:38:37 +0100 Subject: [PATCH 03/45] Fix permission issue and model page --- backend/core/models.py | 3 ++- backend/login/permissions.py | 15 ++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/backend/core/models.py b/backend/core/models.py index 3964587d..a41983d3 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -36,6 +36,7 @@ class DownloadStatus(models.IntegerChoices): label_fetched = models.DateTimeField(null=True, blank=True) created_at = models.DateTimeField(auto_now_add=True) last_modified = models.DateTimeField(auto_now=True) + user = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE) class Label(models.Model): @@ -57,7 +58,7 @@ class ModelStatus(models.IntegerChoices): PUBLISHED = 0 DRAFT = -1 - dataset = models.ForeignKey(Dataset, to_field="id", on_delete=models.CASCADE) + dataset = models.ForeignKey(Dataset, to_field="id", on_delete=models.DO_NOTHING) name = models.CharField(max_length=255) created_at = models.DateTimeField(auto_now_add=True) last_modified = models.DateTimeField(auto_now=True) diff --git a/backend/login/permissions.py b/backend/login/permissions.py index 15cf8ade..d33a5549 100644 --- a/backend/login/permissions.py +++ b/backend/login/permissions.py @@ -27,17 +27,18 @@ def has_object_permission(self, request, view, obj): if request.user.is_staff or request.user.is_superuser: return True ## if the object it is trying to access has user info + if request.user and request.user.is_authenticated: + authenticated_user_allowed_methods = getattr( + view, "authenticated_user_allowed_methods", [] + ) + if request.method in authenticated_user_allowed_methods: + return True + if hasattr(obj, "user"): # in order to change it it needs to be in his/her name if obj.user == request.user: return True - else: - if request.user and request.user.is_authenticated: - authenticated_user_allowed_methods = getattr( - view, "authenticated_user_allowed_methods", [] - ) - if request.method in authenticated_user_allowed_methods: - return True + return False From 8067e163616ac2cf03682d8f9dab794637fa985f Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Fri, 1 Nov 2024 12:42:24 +0100 Subject: [PATCH 04/45] Add user information on the AOI serializer for consistency --- backend/core/serializers.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 22507f2b..b0f1bd23 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -142,6 +142,18 @@ class Meta: "label_status", ) + def create(self, validated_data): + request = self.context.get("request") + if request and hasattr(request, "user"): + validated_data["user"] = request.user + return super().create(validated_data) + + def update(self, instance, validated_data): + request = self.context.get("request") + if request and hasattr(request, "user"): + validated_data["user"] = request.user + return super().update(instance, validated_data) + class FeedbackAOISerializer(GeoFeatureModelSerializer): class Meta: From 4a272791e86d8c10cb0ac10e886ae91f281e7162 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Fri, 1 Nov 2024 12:42:32 +0100 Subject: [PATCH 05/45] Add user in readonly --- backend/core/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/core/serializers.py b/backend/core/serializers.py index b0f1bd23..cb932485 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -140,6 +140,7 @@ class Meta: "last_modified", "label_fetched", "label_status", + "user", ) def create(self, validated_data): From 1569365b95c880c74bd3e802f375537bec53d5c6 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Fri, 1 Nov 2024 12:57:51 +0100 Subject: [PATCH 06/45] Limit title length and description --- backend/core/models.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/backend/core/models.py b/backend/core/models.py index a41983d3..746df4d6 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -14,7 +14,7 @@ class DatasetStatus(models.IntegerChoices): ACTIVE = 0 DRAFT = -1 - name = models.CharField(max_length=255) + name = models.CharField(max_length=50, min_length=5) user = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE) last_modified = models.DateTimeField(auto_now=True) created_at = models.DateTimeField(auto_now_add=True) @@ -49,8 +49,9 @@ class Label(models.Model): class Model(models.Model): BASE_MODEL_CHOICES = ( - ("RAMP", "RAMP"), - ("YOLO", "YOLO"), + ("RAMP", 0), + ("YOLO_V8_V1", 1), + ("YOLO_V8_V2", 2), ) class ModelStatus(models.IntegerChoices): @@ -59,16 +60,14 @@ class ModelStatus(models.IntegerChoices): DRAFT = -1 dataset = models.ForeignKey(Dataset, to_field="id", on_delete=models.DO_NOTHING) - name = models.CharField(max_length=255) + name = models.CharField(max_length=50, min_length=5) created_at = models.DateTimeField(auto_now_add=True) last_modified = models.DateTimeField(auto_now=True) description = models.TextField(max_length=500, null=True, blank=True) user = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE) published_training = models.PositiveIntegerField(null=True, blank=True) status = models.IntegerField(default=-1, choices=ModelStatus.choices) - base_model = models.CharField( - choices=BASE_MODEL_CHOICES, default="RAMP", max_length=10 - ) + base_model = models.IntegerField(choices=BASE_MODEL_CHOICES, default=0) class Training(models.Model): From b5e2e169997ff3dfc156fe2d286d66ee9fb47c9c Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Fri, 1 Nov 2024 12:59:40 +0100 Subject: [PATCH 07/45] Remove min length concept --- backend/core/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/core/models.py b/backend/core/models.py index 746df4d6..94a5b856 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -14,7 +14,7 @@ class DatasetStatus(models.IntegerChoices): ACTIVE = 0 DRAFT = -1 - name = models.CharField(max_length=50, min_length=5) + name = models.CharField(max_length=50) user = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE) last_modified = models.DateTimeField(auto_now=True) created_at = models.DateTimeField(auto_now_add=True) @@ -60,7 +60,7 @@ class ModelStatus(models.IntegerChoices): DRAFT = -1 dataset = models.ForeignKey(Dataset, to_field="id", on_delete=models.DO_NOTHING) - name = models.CharField(max_length=50, min_length=5) + name = models.CharField(max_length=50) created_at = models.DateTimeField(auto_now_add=True) last_modified = models.DateTimeField(auto_now=True) description = models.TextField(max_length=500, null=True, blank=True) From 4532c8bed299f9e211bd656a2706f013fd4c9835 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Fri, 1 Nov 2024 13:02:51 +0100 Subject: [PATCH 08/45] Restored base model field --- backend/core/models.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/core/models.py b/backend/core/models.py index 94a5b856..3244ca83 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -49,9 +49,9 @@ class Label(models.Model): class Model(models.Model): BASE_MODEL_CHOICES = ( - ("RAMP", 0), - ("YOLO_V8_V1", 1), - ("YOLO_V8_V2", 2), + ("RAMP", "RAMP"), + ("YOLO_V8_V1", "YOLO_V8_V1"), + ("YOLO_V8_V2", "YOLO_V8_V2"), ) class ModelStatus(models.IntegerChoices): @@ -67,7 +67,9 @@ class ModelStatus(models.IntegerChoices): user = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE) published_training = models.PositiveIntegerField(null=True, blank=True) status = models.IntegerField(default=-1, choices=ModelStatus.choices) - base_model = models.IntegerField(choices=BASE_MODEL_CHOICES, default=0) + base_model = models.CharField( + choices=BASE_MODEL_CHOICES, default="RAMP", max_length=50 + ) class Training(models.Model): From e884f46c658d4e2d8ce8907882da85b739b1841b Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Fri, 1 Nov 2024 23:42:06 +0100 Subject: [PATCH 09/45] Added yolov8v1 --- backend/core/tasks.py | 604 ++++++++++++++++++++++++------------------ 1 file changed, 343 insertions(+), 261 deletions(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index fb27baaa..260213b9 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -5,11 +5,20 @@ import subprocess import sys import tarfile +import time import traceback from shutil import rmtree from celery import shared_task -from core.models import AOI, Feedback, FeedbackAOI, FeedbackLabel, Label, Training +from core.models import ( + AOI, + Feedback, + FeedbackAOI, + FeedbackLabel, + Label, + Model, + Training, +) from core.serializers import ( AOISerializer, FeedbackAOISerializer, @@ -23,6 +32,7 @@ from django.contrib.gis.geos import GEOSGeometry from django.shortcuts import get_object_or_404 from django.utils import timezone +from predictor import download_imagery, get_start_end_download_coords logger = logging.getLogger(__name__) @@ -32,6 +42,18 @@ DEFAULT_TILE_SIZE = 256 +class print_time: + def __init__(self, name): + self.name = name + + def __enter__(self): + self.start = time.perf_counter() + return self + + def __exit__(self, type, value, traceback): + print(f"{self.name} took {round(time.perf_counter() - self.start, 2)} seconds") + + def xz_folder(folder_path, output_filename, remove_original=False): """ Compresses a folder and its contents into a .tar.xz file and optionally removes the original folder. @@ -66,6 +88,295 @@ def get_file_count(path): return 0 +def prepare_data(training_instance, dataset_id, feedback, zoom_level, source_imagery): + training_input_base_path = os.path.join( + settings.TRAINING_WORKSPACE, f"dataset_{dataset_id}" + ) + training_input_image_source = os.path.join(training_input_base_path, "input") + if os.path.exists(training_input_image_source): + shutil.rmtree(training_input_image_source) + os.makedirs(training_input_image_source) + + if feedback: + aois = FeedbackAOI.objects.filter(training=feedback) + aoi_serializer = FeedbackAOISerializer(aois, many=True) + else: + aois = AOI.objects.filter(dataset=dataset_id) + aoi_serializer = AOISerializer(aois, many=True) + + first_aoi_centroid = aois[0].geom.centroid + training_instance.centroid = first_aoi_centroid + training_instance.save() + + for obj in aois: + bbox_coords = bbox(obj.geom.coords[0]) + for z in zoom_level: + zm_level = z + try: + tile_size = DEFAULT_TILE_SIZE + start, end = get_start_end_download_coords( + bbox_coords, zm_level, tile_size + ) + download_imagery( + start, + end, + zm_level, + base_path=training_input_image_source, + source=source_imagery, + ) + except Exception as ex: + raise ex + if is_dir_empty(training_input_image_source): + raise ValueError("No images found in the area") + + if feedback: + label = FeedbackLabel.objects.filter(feedback_aoi__in=[r.id for r in aois]) + serialized_field = FeedbackLabelFileSerializer(label, many=True) + else: + label = Label.objects.filter(aoi__in=[r.id for r in aois]) + serialized_field = LabelFileSerializer(label, many=True) + + with open( + os.path.join(training_input_image_source, "labels.geojson"), + "w", + encoding="utf-8", + ) as f: + f.write(json.dumps(serialized_field.data)) + + return training_input_image_source, aoi_serializer, serialized_field + + +def ramp_model_training( + training_instance, + dataset_id, + training_input_image_source, + serialized_field, + aoi_serializer, + epochs, + batch_size, + freeze_layers, + multimasks, + input_contact_spacing, + input_boundary_width, +): + import hot_fair_utilities + import ramp.utils + import tensorflow as tf + from hot_fair_utilities import preprocess + from hot_fair_utilities.training import run_feedback + from hot_fair_utilities.training.ramp import train + + base_path = os.path.join(settings.RAMP_HOME, "ramp-data", str(dataset_id)) + if os.path.exists(base_path): + rmtree(base_path) + destination_image_input = os.path.join(base_path, "input") + + if not os.path.exists(training_input_image_source): + raise ValueError( + "Training folder has not been created, Build the dataset first /dataset/build/" + ) + if os.path.exists(destination_image_input): + shutil.rmtree(destination_image_input) + shutil.copytree(training_input_image_source, destination_image_input) + + model_input_image_path = f"{base_path}/input" + preprocess_output = f"/{base_path}/preprocessed" + + preprocess( + input_path=model_input_image_path, + output_path=preprocess_output, + rasterize=True, + rasterize_options=["binary"], + georeference_images=True, + multimasks=multimasks, + input_contact_spacing=input_contact_spacing, + input_boundary_width=input_boundary_width, + ) + training_instance.chips_length = get_file_count( + os.path.join(preprocess_output, "chips") + ) + training_instance.save() + + train_output = f"{base_path}/train" + final_accuracy, final_model_path = train( + input_path=preprocess_output, + output_path=train_output, + epoch_size=epochs, + batch_size=batch_size, + model="ramp", + model_home=os.environ["RAMP_HOME"], + freeze_layers=freeze_layers, + multimasks=multimasks, + ) + + output_path = os.path.join( + training_input_image_source, "output", f"training_{training_instance.id}" + ) + if os.path.exists(output_path): + shutil.rmtree(output_path) + shutil.copytree(final_model_path, os.path.join(output_path, "checkpoint.tf")) + shutil.copytree(preprocess_output, os.path.join(output_path, "preprocessed")) + + graph_output_path = f"{base_path}/train/graphs" + shutil.copytree(graph_output_path, os.path.join(output_path, "graphs")) + + with open(os.path.join(output_path, "labels.geojson"), "w", encoding="utf-8") as f: + f.write(json.dumps(serialized_field.data)) + + with open(os.path.join(output_path, "aois.geojson"), "w", encoding="utf-8") as f: + f.write(json.dumps(aoi_serializer.data)) + + tippecanoe_command = f"""tippecanoe -o {os.path.join(output_path,"meta.pmtiles")} -Z7 -z18 -L aois:{ os.path.join(output_path, "aois.geojson")} -L labels:{os.path.join(output_path, "labels.geojson")} --force --read-parallel -rg --drop-densest-as-needed""" + try: + result = subprocess.run( + tippecanoe_command, shell=True, check=True, capture_output=True + ) + logging.info(result.stdout.decode("utf-8")) + except subprocess.CalledProcessError as ex: + logger.error(ex.output) + raise ex + + shutil.copyfile( + os.path.join(output_path, "aois.geojson"), + os.path.join(preprocess_output, "aois.geojson"), + ) + shutil.copyfile( + os.path.join(output_path, "labels.geojson"), + os.path.join(preprocess_output, "labels.geojson"), + ) + xz_folder( + preprocess_output, + os.path.join(output_path, "preprocessed.tar.xz"), + remove_original=True, + ) + shutil.rmtree(base_path) + training_instance.accuracy = float(final_accuracy) + training_instance.finished_at = timezone.now() + training_instance.status = "FINISHED" + training_instance.save() + response = { + "accuracy": float(final_accuracy), + "tiles_path": os.path.join(output_path, "meta.pmtiles"), + "model_path": os.path.join(output_path, "checkpoint.h5"), + "graph_path": os.path.join(output_path, "graphs"), + } + return response + + +def yolo_model_training( + training_instance, + dataset_id, + training_input_image_source, + serialized_field, + aoi_serializer, + epochs, + batch_size, + multimasks, +): + from hot_fair_utilities import preprocess + from hot_fair_utilities.preprocessing.yolo_v8_v1.yolo_format import yolo_format + from hot_fair_utilities.training.yolo_v8_v1.train import train as train_yolo + + base_path = os.path.join(settings.YOLO_HOME, "yolo-data", str(dataset_id)) + if os.path.exists(base_path): + rmtree(base_path) + destination_image_input = os.path.join(base_path, "input") + + if not os.path.exists(training_input_image_source): + raise ValueError( + "Training folder has not been created, Build the dataset first /dataset/build/" + ) + if os.path.exists(destination_image_input): + shutil.rmtree(destination_image_input) + shutil.copytree(training_input_image_source, destination_image_input) + + model_input_image_path = f"{base_path}/input" + preprocess_output = f"/{base_path}/preprocessed" + + preprocess( + input_path=model_input_image_path, + output_path=preprocess_output, + rasterize=True, + rasterize_options=["binary"], + georeference_images=True, + multimasks=multimasks, + ) + training_instance.chips_length = get_file_count( + os.path.join(preprocess_output, "chips") + ) + training_instance.save() + + final_accuracy = 80 # TODO: Replace with actual training logic + + with print_time("yolo conversion"): + yolo_format( + preprocessed_dirs=preprocess_output, + yolo_dir=f"{base_path}/yolo", + multimask=True, + p_val=0.05, + ) + output_model_path = train_yolo( + data=f"{base_path}", + weights=f"{os.getcwd()}/yolov8s_v1-seg-best.pt", + epochs=epochs, + batch_size=batch_size, + pc=2.0, + ) + + output_path = os.path.join( + training_input_image_source, "output", f"training_{training_instance.id}" + ) + if os.path.exists(output_path): + shutil.rmtree(output_path) + shutil.copytree(output_model_path, os.path.join(output_path, "checkpoint.pt")) + shutil.copytree(preprocess_output, os.path.join(output_path, "preprocessed")) + + graph_output_path = f"{base_path}/train/graphs" + shutil.copytree(graph_output_path, os.path.join(output_path, "graphs")) + + with open(os.path.join(output_path, "labels.geojson"), "w", encoding="utf-8") as f: + f.write(json.dumps(serialized_field.data)) + + with open(os.path.join(output_path, "aois.geojson"), "w", encoding="utf-8") as f: + f.write(json.dumps(aoi_serializer.data)) + + tippecanoe_command = f"""tippecanoe -o {os.path.join(output_path,"meta.pmtiles")} -Z7 -z18 -L aois:{ os.path.join(output_path, "aois.geojson")} -L labels:{os.path.join(output_path, "labels.geojson")} --force --read-parallel -rg --drop-densest-as-needed""" + try: + result = subprocess.run( + tippecanoe_command, shell=True, check=True, capture_output=True + ) + logging.info(result.stdout.decode("utf-8")) + except subprocess.CalledProcessError as ex: + logger.error(ex.output) + raise ex + + shutil.copyfile( + os.path.join(output_path, "aois.geojson"), + os.path.join(preprocess_output, "aois.geojson"), + ) + shutil.copyfile( + os.path.join(output_path, "labels.geojson"), + os.path.join(preprocess_output, "labels.geojson"), + ) + xz_folder( + preprocess_output, + os.path.join(output_path, "preprocessed.tar.xz"), + remove_original=True, + ) + shutil.rmtree(base_path) + training_instance.accuracy = float(final_accuracy) + training_instance.finished_at = timezone.now() + training_instance.status = "FINISHED" + training_instance.save() + response = { + "accuracy": float(final_accuracy), + "tiles_path": os.path.join(output_path, "meta.pmtiles"), + "model_path": os.path.join(output_path, "checkpoint.pt"), + "graph_path": os.path.join(output_path, "graphs"), + } + return response + + @shared_task def train_model( dataset_id, @@ -80,279 +391,50 @@ def train_model( input_contact_spacing=8, input_boundary_width=3, ): - # importing them here so that it won't be necessary when sending tasks ( api only) - import hot_fair_utilities - import ramp.utils - import tensorflow as tf - from hot_fair_utilities import preprocess - from hot_fair_utilities.training import run_feedback - from hot_fair_utilities.training.ramp import train - from predictor import download_imagery, get_start_end_download_coords training_instance = get_object_or_404(Training, id=training_id) + model_instance = get_object_or_404(Model, id=training_instance.model.id) + training_instance.status = "RUNNING" training_instance.started_at = timezone.now() training_instance.save() - if settings.RAMP_HOME is None: + + if model_instance.base_model == "YOLO_V8_V1" and settings.YOLO_HOME is None: + raise ValueError("YOLO Home is not configured") + elif model_instance.base_model != "YOLO_V8_V1" and settings.RAMP_HOME is None: raise ValueError("Ramp Home is not configured") try: - ## -----------IMAGE DOWNLOADER--------- - os.makedirs(settings.LOG_PATH, exist_ok=True) - if training_instance.task_id is None or training_instance.task_id.strip() == "": - training_instance.task_id = train_model.request.id - training_instance.save() - log_file = os.path.join( - settings.LOG_PATH, f"run_{train_model.request.id}_log.txt" + training_input_image_source, aoi_serializer, serialized_field = prepare_data( + training_instance, dataset_id, feedback, zoom_level, source_imagery ) - with open(log_file, "w") as f: - # redirect stdout to the log file - sys.stdout = f - training_input_base_path = os.path.join( - settings.TRAINING_WORKSPACE, f"dataset_{dataset_id}" - ) - training_input_image_source = os.path.join( - training_input_base_path, "input" - ) - if os.path.exists(training_input_image_source): # always build dataset - shutil.rmtree(training_input_image_source) - os.makedirs(training_input_image_source) - if feedback: - try: - aois = FeedbackAOI.objects.filter(training=feedback) - aoi_serializer = FeedbackAOISerializer(aois, many=True) - - except FeedbackAOI.DoesNotExist: - raise ValueError( - f"No Feedback AOI is attached with supplied training id:{dataset_id}, Create AOI first", - ) - - else: - try: - aois = AOI.objects.filter(dataset=dataset_id) - aoi_serializer = AOISerializer(aois, many=True) - - except AOI.DoesNotExist: - raise ValueError( - f"No AOI is attached with supplied dataset id:{dataset_id}, Create AOI first", - ) - first_aoi_centroid = aois[0].geom.centroid - training_instance.centroid = first_aoi_centroid - training_instance.save() - - for obj in aois: - bbox_coords = bbox(obj.geom.coords[0]) - for z in zoom_level: - zm_level = z - print( - f"""Running Download process for - aoi : {obj.id} - dataset : {dataset_id} , zoom : {zm_level}""" - ) - try: - tile_size = DEFAULT_TILE_SIZE # by default - - start, end = get_start_end_download_coords( - bbox_coords, zm_level, tile_size - ) - # start downloading - download_imagery( - start, - end, - zm_level, - base_path=training_input_image_source, - source=source_imagery, - ) - - except Exception as ex: - raise ex - if is_dir_empty(training_input_image_source): - raise ValueError("No images found in the area") - - ## -----------LABEL GENERATOR--------- - logging.info("Label Generator started") - aoi_list = [r.id for r in aois] - logging.info(aoi_list) - - if feedback: - label = FeedbackLabel.objects.filter(feedback_aoi__in=aoi_list) - logging.info(label) - - serialized_field = FeedbackLabelFileSerializer(label, many=True) - else: - label = Label.objects.filter(aoi__in=aoi_list) - serialized_field = LabelFileSerializer(label, many=True) - - with open( - os.path.join(training_input_image_source, "labels.geojson"), - "w", - encoding="utf-8", - ) as f: - f.write(json.dumps(serialized_field.data)) - - ## --------- Data Preparation ---------- - base_path = os.path.join(settings.RAMP_HOME, "ramp-data", str(dataset_id)) - # Check if the path exists - if os.path.exists(base_path): - # Delete the directory and its contents - rmtree(base_path) - destination_image_input = os.path.join(base_path, "input") - - logging.info(training_input_image_source) - if not os.path.exists(training_input_image_source): - raise ValueError( - "Training folder has not been created at , Build the dataset first /dataset/build/" - ) - if os.path.exists(destination_image_input): - shutil.rmtree(destination_image_input) - shutil.copytree(training_input_image_source, destination_image_input) - # preprocess - model_input_image_path = f"{base_path}/input" - preprocess_output = f"/{base_path}/preprocessed" - - if multimasks: - logger.info( - "Using multiple masks for training : background, footprint, boundary, contact" - ) - else: - logger.info("Using binary masks for training : background, footprint") - preprocess( - input_path=model_input_image_path, - output_path=preprocess_output, - rasterize=True, - rasterize_options=["binary"], - georeference_images=True, - multimasks=multimasks, - input_contact_spacing=input_contact_spacing, - input_boundary_width=input_boundary_width, - ) - training_instance.chips_length = get_file_count( - os.path.join(preprocess_output, "chips") - ) - training_instance.save() - - # train - - train_output = f"{base_path}/train" - if feedback: - final_accuracy, final_model_path = run_feedback( - input_path=preprocess_output, - output_path=train_output, - feedback_base_model=os.path.join( - settings.TRAINING_WORKSPACE, - f"dataset_{dataset_id}", - "output", - f"training_{feedback}", - "checkpoint.tf", - ), - model_home=os.environ["RAMP_HOME"], - epoch_size=epochs, - multimasks=multimasks, - batch_size=batch_size, - freeze_layers=freeze_layers, - ) - else: - final_accuracy, final_model_path = train( - input_path=preprocess_output, - output_path=train_output, - epoch_size=epochs, - batch_size=batch_size, - model="ramp", - model_home=os.environ["RAMP_HOME"], - freeze_layers=freeze_layers, - multimasks=multimasks, - ) - - # copy final model to output - output_path = os.path.join( - training_input_base_path, "output", f"training_{training_id}" - ) - if os.path.exists(output_path): - shutil.rmtree(output_path) - shutil.copytree( - final_model_path, os.path.join(output_path, "checkpoint.tf") - ) - - # shutil.copytree( - # preprocess_output, os.path.join(output_path, "preprocessed") - # ) - - graph_output_path = f"{base_path}/train/graphs" - shutil.copytree(graph_output_path, os.path.join(output_path, "graphs")) - - # convert model to hdf5 for faster inference - model = tf.keras.models.load_model( - os.path.join(output_path, "checkpoint.tf") - ) - # Save the model in HDF5 format - model.save(os.path.join(output_path, "checkpoint.h5")) - - logger.info(model.inputs) - logger.info(model.outputs) - - # Convert the model to tflite for android/ios. - converter = tf.lite.TFLiteConverter.from_keras_model(model) - tflite_model = converter.convert() - - # Save the model. - with open(os.path.join(output_path, "checkpoint.tflite"), "wb") as f: - f.write(tflite_model) - - # dump labels to output folder as well - with open( - os.path.join(output_path, "labels.geojson"), - "w", - encoding="utf-8", - ) as f: - f.write(json.dumps(serialized_field.data)) - - # dump used aois as featurecollection in output - with open( - os.path.join(output_path, "aois.geojson"), - "w", - encoding="utf-8", - ) as f: - f.write(json.dumps(aoi_serializer.data)) - - tippecanoe_command = f"""tippecanoe -o {os.path.join(output_path,"meta.pmtiles")} -Z7 -z18 -L aois:{ os.path.join(output_path, "aois.geojson")} -L labels:{os.path.join(output_path, "labels.geojson")} --force --read-parallel -rg --drop-densest-as-needed""" - logging.info("Starting to generate vector tiles for aois and labels") - try: - result = subprocess.run( - tippecanoe_command, shell=True, check=True, capture_output=True - ) - logging.info(result.stdout.decode("utf-8")) - except subprocess.CalledProcessError as ex: - logger.error(ex.output) - raise ex - logging.info("Vector tile generation done !") - - # copy aois and labels to preprocess output before compressing it to tar - shutil.copyfile( - os.path.join(output_path, "aois.geojson"), - os.path.join(preprocess_output, "aois.geojson"), - ) - shutil.copyfile( - os.path.join(output_path, "labels.geojson"), - os.path.join(preprocess_output, "labels.geojson"), + if model_instance.base_model == "YOLO_V8_V1": + response = yolo_model_training( + training_instance, + dataset_id, + training_input_image_source, + serialized_field, + aoi_serializer, + epochs, + batch_size, + multimasks, ) - xz_folder( - preprocess_output, - os.path.join(output_path, "preprocessed.tar.xz"), - remove_original=True, + else: + response = ramp_model_training( + training_instance, + dataset_id, + training_input_image_source, + serialized_field, + aoi_serializer, + epochs, + batch_size, + freeze_layers, + multimasks, + input_contact_spacing, + input_boundary_width, ) - # now remove the ramp-data all our outputs are copied to our training workspace - shutil.rmtree(base_path) - training_instance.accuracy = float(final_accuracy) - training_instance.finished_at = timezone.now() - training_instance.status = "FINISHED" - training_instance.save() - response = {} - response["accuracy"] = float(final_accuracy) - response["tiles_path"] = os.path.join(output_path, "meta.pmtiles") - response["model_path"] = os.path.join(output_path, "checkpoint.h5") - response["graph_path"] = os.path.join(output_path, "graphs") - sys.stdout = sys.__stdout__ logger.info(f"Training task {training_id} completed successfully") return response From 72f0cd457162f5f669efb734a8aba5b86f6ac117 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Sat, 2 Nov 2024 13:28:33 +0100 Subject: [PATCH 10/45] Updated logic for both version of yolo --- backend/core/tasks.py | 61 +++++++++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 260213b9..ed9deb0c 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -272,10 +272,17 @@ def yolo_model_training( epochs, batch_size, multimasks, + model="YOLO_V8_V1", ): from hot_fair_utilities import preprocess - from hot_fair_utilities.preprocessing.yolo_v8_v1.yolo_format import yolo_format - from hot_fair_utilities.training.yolo_v8_v1.train import train as train_yolo + from hot_fair_utilities.preprocessing.yolo_v8_v1.yolo_format import ( + yolo_format as yolo_format_v1, + ) + from hot_fair_utilities.preprocessing.yolo_v8_v2.yolo_format import ( + yolo_format as yolo_format_v2, + ) + from hot_fair_utilities.training.yolo_v8_v1.train import train as train_yolo_v1 + from hot_fair_utilities.training.yolo_v8_v2.train import train as train_yolo_v2 base_path = os.path.join(settings.YOLO_HOME, "yolo-data", str(dataset_id)) if os.path.exists(base_path): @@ -300,6 +307,7 @@ def yolo_model_training( rasterize_options=["binary"], georeference_images=True, multimasks=multimasks, + epsg=4326 if model == "YOLO_V8_V2" else 3857, ) training_instance.chips_length = get_file_count( os.path.join(preprocess_output, "chips") @@ -307,21 +315,41 @@ def yolo_model_training( training_instance.save() final_accuracy = 80 # TODO: Replace with actual training logic - + yolo_data_dir = os.path.join(base_path, model) with print_time("yolo conversion"): - yolo_format( - preprocessed_dirs=preprocess_output, - yolo_dir=f"{base_path}/yolo", - multimask=True, - p_val=0.05, + if model == "YOLO_V8_V1": + yolo_format_v1( + preprocessed_dirs=preprocess_output, + yolo_dir=yolo_data_dir, + multimask=True, + p_val=0.05, + ) + else: + yolo_format_v2( + input_path=preprocess_output, + output_path=yolo_data_dir, + ) + if model == "YOLO_V8_V1": + output_model_path = train_yolo_v1( + data=f"{base_path}", + weights=os.path.join(settings.YOLO_HOME, "yolov8s_v1-seg-best.pt"), + epochs=epochs, + batch_size=batch_size, + pc=2.0, + output_path=yolo_data_dir, + dataset_yaml_path=os.path.join(yolo_data_dir, "yolo_dataset.yaml"), + ) + else: + output_model_path = train_yolo_v2( + data=f"{base_path}", + weights=f"{os.getcwd()}/yolov8s_v2-seg.pt", + gpu="cpu", + epochs=2, + batch_size=16, + pc=2.0, + output_path=yolo_data_dir, + dataset_yaml_path=os.path.join(yolo_data_dir, "yolo_dataset.yaml"), ) - output_model_path = train_yolo( - data=f"{base_path}", - weights=f"{os.getcwd()}/yolov8s_v1-seg-best.pt", - epochs=epochs, - batch_size=batch_size, - pc=2.0, - ) output_path = os.path.join( training_input_image_source, "output", f"training_{training_instance.id}" @@ -409,7 +437,7 @@ def train_model( training_instance, dataset_id, feedback, zoom_level, source_imagery ) - if model_instance.base_model == "YOLO_V8_V1": + if model_instance.base_model in ("YOLO_V8_V1", "YOLO_V8_V2"): response = yolo_model_training( training_instance, dataset_id, @@ -419,6 +447,7 @@ def train_model( epochs, batch_size, multimasks, + model=model_instance.base_model, ) else: response = ramp_model_training( From 2314e8ee5510669caec85669127931bfa39a61e1 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 4 Nov 2024 11:39:36 +0100 Subject: [PATCH 11/45] Added Yolo Home for the settings --- backend/aiproject/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/aiproject/settings.py b/backend/aiproject/settings.py index 106104eb..4d350bf3 100644 --- a/backend/aiproject/settings.py +++ b/backend/aiproject/settings.py @@ -215,6 +215,8 @@ RAMP_HOME = env("RAMP_HOME", default=None) if RAMP_HOME: os.environ["RAMP_HOME"] = RAMP_HOME +YOLO_HOME = env("YOLO_HOME") + # training workspace TRAINING_WORKSPACE = env( From 21f094e0d908724fc3fab497ba9d1fdff4d763fd Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 4 Nov 2024 12:47:13 +0100 Subject: [PATCH 12/45] import run feedback from ramp --- backend/core/tasks.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index ed9deb0c..b592c3f7 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -163,8 +163,7 @@ def ramp_model_training( import ramp.utils import tensorflow as tf from hot_fair_utilities import preprocess - from hot_fair_utilities.training import run_feedback - from hot_fair_utilities.training.ramp import train + from hot_fair_utilities.training.ramp import run_feedback, train base_path = os.path.join(settings.RAMP_HOME, "ramp-data", str(dataset_id)) if os.path.exists(base_path): @@ -342,7 +341,7 @@ def yolo_model_training( else: output_model_path = train_yolo_v2( data=f"{base_path}", - weights=f"{os.getcwd()}/yolov8s_v2-seg.pt", + weights=os.path.join(settings.YOLO_HOME, "yolov8s_v2-seg.pt"), gpu="cpu", epochs=2, batch_size=16, From dfeac3cd8c27452f533c10f775969f59269ef8c0 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 4 Nov 2024 14:18:51 +0100 Subject: [PATCH 13/45] Added copy file for the checkpoint point as its not dir for yolo --- backend/core/tasks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index b592c3f7..61789349 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -342,7 +342,6 @@ def yolo_model_training( output_model_path = train_yolo_v2( data=f"{base_path}", weights=os.path.join(settings.YOLO_HOME, "yolov8s_v2-seg.pt"), - gpu="cpu", epochs=2, batch_size=16, pc=2.0, @@ -355,7 +354,7 @@ def yolo_model_training( ) if os.path.exists(output_path): shutil.rmtree(output_path) - shutil.copytree(output_model_path, os.path.join(output_path, "checkpoint.pt")) + shutil.copyfile(output_model_path, os.path.join(output_path, "checkpoint.pt")) shutil.copytree(preprocess_output, os.path.join(output_path, "preprocessed")) graph_output_path = f"{base_path}/train/graphs" From a17cbe0d6139d6d9236f7daced7e7aae10967ed8 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 4 Nov 2024 14:40:59 +0100 Subject: [PATCH 14/45] Fix bug on file creation for dir --- backend/core/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 61789349..1c076ecf 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -354,6 +354,7 @@ def yolo_model_training( ) if os.path.exists(output_path): shutil.rmtree(output_path) + os.makedirs(output_path) shutil.copyfile(output_model_path, os.path.join(output_path, "checkpoint.pt")) shutil.copytree(preprocess_output, os.path.join(output_path, "preprocessed")) From 04b187f4deee5430de7ed69998c01839faf9afee Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 4 Nov 2024 15:01:30 +0100 Subject: [PATCH 15/45] Update graph output handling in YOLO model training --- backend/core/tasks.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 1c076ecf..68e16091 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -358,8 +358,16 @@ def yolo_model_training( shutil.copyfile(output_model_path, os.path.join(output_path, "checkpoint.pt")) shutil.copytree(preprocess_output, os.path.join(output_path, "preprocessed")) - graph_output_path = f"{base_path}/train/graphs" - shutil.copytree(graph_output_path, os.path.join(output_path, "graphs")) + graph_output_path = os.path.join(os.path.dirname(output_model_path), "results.png") + os.makedirs(os.path.join(output_path, "graphs"), exist_ok=True) + shutil.copyfile( + graph_output_path, + os.path.join( + output_path, + "graphs", + "training_validation_sparse_categorical_accuracy.png", ### TODO : replace this with actual graph that will be decided + ), + ) with open(os.path.join(output_path, "labels.geojson"), "w", encoding="utf-8") as f: f.write(json.dumps(serialized_field.data)) From 4a55ccb32ac44d0248f81073d897a0af81102199 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 4 Nov 2024 15:07:08 +0100 Subject: [PATCH 16/45] Refactor graph output path handling to use pathlib for improved path management --- backend/core/tasks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 68e16091..e7c4e428 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -1,6 +1,7 @@ import json import logging import os +import pathlib import shutil import subprocess import sys @@ -358,7 +359,9 @@ def yolo_model_training( shutil.copyfile(output_model_path, os.path.join(output_path, "checkpoint.pt")) shutil.copytree(preprocess_output, os.path.join(output_path, "preprocessed")) - graph_output_path = os.path.join(os.path.dirname(output_model_path), "results.png") + graph_output_path = os.path.join( + pathlib.Path(os.path.dirname(output_model_path)).parent, "results.png" + ) os.makedirs(os.path.join(output_path, "graphs"), exist_ok=True) shutil.copyfile( graph_output_path, From cca0fc7963a5101b5b798060b7a67a60a2977b7e Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 4 Nov 2024 15:30:47 +0100 Subject: [PATCH 17/45] Add debug print statement for output path in YOLO model training --- backend/core/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index e7c4e428..9f08b03d 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -355,6 +355,7 @@ def yolo_model_training( ) if os.path.exists(output_path): shutil.rmtree(output_path) + print(output_path) os.makedirs(output_path) shutil.copyfile(output_model_path, os.path.join(output_path, "checkpoint.pt")) shutil.copytree(preprocess_output, os.path.join(output_path, "preprocessed")) From 70cda47850334740f3306758d663c3a91a98322f Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 4 Nov 2024 15:36:27 +0100 Subject: [PATCH 18/45] Refactor output path construction in YOLO model training to use pathlib --- backend/core/tasks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 9f08b03d..9eb3309c 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -351,11 +351,13 @@ def yolo_model_training( ) output_path = os.path.join( - training_input_image_source, "output", f"training_{training_instance.id}" + pathlib.Path(training_input_image_source).parent, + "output", + f"training_{training_instance.id}", ) if os.path.exists(output_path): shutil.rmtree(output_path) - print(output_path) + # print(output_path) os.makedirs(output_path) shutil.copyfile(output_model_path, os.path.join(output_path, "checkpoint.pt")) shutil.copytree(preprocess_output, os.path.join(output_path, "preprocessed")) From c02829446bce3583fa39d255d32a123d6d2a923d Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 5 Nov 2024 11:18:07 +0100 Subject: [PATCH 19/45] added final accuracy metrics for the IOU in fAIr --- backend/core/tasks.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 9eb3309c..afd3dd1c 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -89,6 +89,25 @@ def get_file_count(path): return 0 +def get_yolo_iou_metrics(model_path): + import ultralytics + + model_val = ultralytics.YOLO(model_path) + model_val_metrics = ( + model_val.val().results_dict + ) ### B and M denotes bounding box and mask respectively + # print(metrics) + print(model_val_metrics) + + iou_accuracy = 1 / ( + 1 / model_val_metrics["metrics/precision(M)"] + + 1 / model_val_metrics["metrics/recall(M)"] + - 1 + ) # ref here https://github.com/ultralytics/ultralytics/issues/9984#issuecomment-2422551315 + final_accuracy = iou_accuracy * 100 + return final_accuracy + + def prepare_data(training_instance, dataset_id, feedback, zoom_level, source_imagery): training_input_base_path = os.path.join( settings.TRAINING_WORKSPACE, f"dataset_{dataset_id}" @@ -314,7 +333,6 @@ def yolo_model_training( ) training_instance.save() - final_accuracy = 80 # TODO: Replace with actual training logic yolo_data_dir = os.path.join(base_path, model) with print_time("yolo conversion"): if model == "YOLO_V8_V1": @@ -359,6 +377,8 @@ def yolo_model_training( shutil.rmtree(output_path) # print(output_path) os.makedirs(output_path) + final_accuracy = get_yolo_iou_metrics(output_model_path) + shutil.copyfile(output_model_path, os.path.join(output_path, "checkpoint.pt")) shutil.copytree(preprocess_output, os.path.join(output_path, "preprocessed")) From 326e83ac0443333b33bc20c0873fdad206a69474 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 5 Nov 2024 14:06:59 +0100 Subject: [PATCH 20/45] Refactor YOLO model training to calculate and return final accuracy metrics; update output file names for clarity --- backend/core/tasks.py | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index afd3dd1c..fe6bba8a 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -89,25 +89,6 @@ def get_file_count(path): return 0 -def get_yolo_iou_metrics(model_path): - import ultralytics - - model_val = ultralytics.YOLO(model_path) - model_val_metrics = ( - model_val.val().results_dict - ) ### B and M denotes bounding box and mask respectively - # print(metrics) - print(model_val_metrics) - - iou_accuracy = 1 / ( - 1 / model_val_metrics["metrics/precision(M)"] - + 1 / model_val_metrics["metrics/recall(M)"] - - 1 - ) # ref here https://github.com/ultralytics/ultralytics/issues/9984#issuecomment-2422551315 - final_accuracy = iou_accuracy * 100 - return final_accuracy - - def prepare_data(training_instance, dataset_id, feedback, zoom_level, source_imagery): training_input_base_path = os.path.join( settings.TRAINING_WORKSPACE, f"dataset_{dataset_id}" @@ -348,7 +329,7 @@ def yolo_model_training( output_path=yolo_data_dir, ) if model == "YOLO_V8_V1": - output_model_path = train_yolo_v1( + output_model_path, final_accuracy = train_yolo_v1( data=f"{base_path}", weights=os.path.join(settings.YOLO_HOME, "yolov8s_v1-seg-best.pt"), epochs=epochs, @@ -358,7 +339,7 @@ def yolo_model_training( dataset_yaml_path=os.path.join(yolo_data_dir, "yolo_dataset.yaml"), ) else: - output_model_path = train_yolo_v2( + output_model_path, final_accuracy = train_yolo_v2( data=f"{base_path}", weights=os.path.join(settings.YOLO_HOME, "yolov8s_v2-seg.pt"), epochs=2, @@ -377,13 +358,12 @@ def yolo_model_training( shutil.rmtree(output_path) # print(output_path) os.makedirs(output_path) - final_accuracy = get_yolo_iou_metrics(output_model_path) shutil.copyfile(output_model_path, os.path.join(output_path, "checkpoint.pt")) shutil.copytree(preprocess_output, os.path.join(output_path, "preprocessed")) graph_output_path = os.path.join( - pathlib.Path(os.path.dirname(output_model_path)).parent, "results.png" + pathlib.Path(os.path.dirname(output_model_path)).parent, "iou_chart.png" ) os.makedirs(os.path.join(output_path, "graphs"), exist_ok=True) shutil.copyfile( @@ -391,7 +371,7 @@ def yolo_model_training( os.path.join( output_path, "graphs", - "training_validation_sparse_categorical_accuracy.png", ### TODO : replace this with actual graph that will be decided + "training_accuracy.png", ### TODO : replace this with actual graph that will be decided ), ) From f9e3c129f0dea8aeeef129cbe9a859b23209b03a Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Sun, 10 Nov 2024 08:09:47 +0100 Subject: [PATCH 21/45] Disable pagination for LabelViewSet to streamline API responses --- backend/core/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/core/views.py b/backend/core/views.py index cbfc9008..a7d09721 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -333,6 +333,7 @@ class LabelViewSet(viewsets.ModelViewSet): queryset = Label.objects.all() serializer_class = LabelSerializer # connecting serializer bbox_filter_field = "geom" + pagination_class = None filter_backends = ( InBBoxFilter, # it will take bbox like this api/v1/label/?in_bbox=-90,29,-89,35 , # TMSTileFilter, # will serve as tms tiles https://wiki.openstreetmap.org/wiki/TMS , use like this ?tile=8/100/200 z/x/y which is equivalent to filtering on the bbox (-39.37500,-71.07406,-37.96875,-70.61261) # Note that the tile address start in the upper left, not the lower left origin used by some implementations. From ae7b0fbd7f7562727ea18b4b26d3688057405b02 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Sun, 10 Nov 2024 08:11:24 +0100 Subject: [PATCH 22/45] Enable TMSTileFilter in LabelViewSet for enhanced tile-based filtering --- backend/core/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/views.py b/backend/core/views.py index a7d09721..a48cea75 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -336,7 +336,7 @@ class LabelViewSet(viewsets.ModelViewSet): pagination_class = None filter_backends = ( InBBoxFilter, # it will take bbox like this api/v1/label/?in_bbox=-90,29,-89,35 , - # TMSTileFilter, # will serve as tms tiles https://wiki.openstreetmap.org/wiki/TMS , use like this ?tile=8/100/200 z/x/y which is equivalent to filtering on the bbox (-39.37500,-71.07406,-37.96875,-70.61261) # Note that the tile address start in the upper left, not the lower left origin used by some implementations. + TMSTileFilter, # will serve as tms tiles https://wiki.openstreetmap.org/wiki/TMS , use like this ?tile=8/100/200 z/x/y which is equivalent to filtering on the bbox (-39.37500,-71.07406,-37.96875,-70.61261) # Note that the tile address start in the upper left, not the lower left origin used by some implementations. DjangoFilterBackend, ) bbox_filter_include_overlapping = ( From 5bf2caa87823d098f3daf350951531d85fec5686 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 12 Nov 2024 13:01:57 +0000 Subject: [PATCH 23/45] Added yolo preprocess output of yolo --- backend/core/tasks.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index fe6bba8a..af14092d 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -210,7 +210,9 @@ def ramp_model_training( ) output_path = os.path.join( - training_input_image_source, "output", f"training_{training_instance.id}" + pathlib.Path(training_input_image_source).parent, + "output", + f"training_{training_instance.id}", ) if os.path.exists(output_path): shutil.rmtree(output_path) @@ -361,6 +363,10 @@ def yolo_model_training( shutil.copyfile(output_model_path, os.path.join(output_path, "checkpoint.pt")) shutil.copytree(preprocess_output, os.path.join(output_path, "preprocessed")) + os.makedirs(os.path.join(output_path,model),exist_ok=True) + shutil.copytree(preprocess_output, os.path.join(output_path,model, "images")) + shutil.copytree(preprocess_output, os.path.join(output_path,model, "labels")) + shutil.copytree(preprocess_output, os.path.join(output_path,model, "yolo_dataset.yaml")) graph_output_path = os.path.join( pathlib.Path(os.path.dirname(output_model_path)).parent, "iou_chart.png" From 3eb05260e09a3b003330b58deaf584e57bab5152 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 12 Nov 2024 14:00:28 +0000 Subject: [PATCH 24/45] Configure logging for the model training --- backend/core/tasks.py | 73 +++++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index af14092d..e3c6c74c 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -445,6 +445,14 @@ def train_model( training_instance.status = "RUNNING" training_instance.started_at = timezone.now() training_instance.save() + os.makedirs(settings.LOG_PATH, exist_ok=True) + if training_instance.task_id is None or training_instance.task_id.strip() == "": + training_instance.task_id = train_model.request.id + training_instance.save() + log_file = os.path.join( + settings.LOG_PATH, f"run_{train_model.request.id}_log.txt" + ) + if model_instance.base_model == "YOLO_V8_V1" and settings.YOLO_HOME is None: raise ValueError("YOLO Home is not configured") @@ -452,39 +460,42 @@ def train_model( raise ValueError("Ramp Home is not configured") try: - training_input_image_source, aoi_serializer, serialized_field = prepare_data( - training_instance, dataset_id, feedback, zoom_level, source_imagery - ) - - if model_instance.base_model in ("YOLO_V8_V1", "YOLO_V8_V2"): - response = yolo_model_training( - training_instance, - dataset_id, - training_input_image_source, - serialized_field, - aoi_serializer, - epochs, - batch_size, - multimasks, - model=model_instance.base_model, - ) - else: - response = ramp_model_training( - training_instance, - dataset_id, - training_input_image_source, - serialized_field, - aoi_serializer, - epochs, - batch_size, - freeze_layers, - multimasks, - input_contact_spacing, - input_boundary_width, + with open(log_file, "w") as f: + # redirect stdout to the log file + sys.stdout = f + training_input_image_source, aoi_serializer, serialized_field = prepare_data( + training_instance, dataset_id, feedback, zoom_level, source_imagery ) - logger.info(f"Training task {training_id} completed successfully") - return response + if model_instance.base_model in ("YOLO_V8_V1", "YOLO_V8_V2"): + response = yolo_model_training( + training_instance, + dataset_id, + training_input_image_source, + serialized_field, + aoi_serializer, + epochs, + batch_size, + multimasks, + model=model_instance.base_model, + ) + else: + response = ramp_model_training( + training_instance, + dataset_id, + training_input_image_source, + serialized_field, + aoi_serializer, + epochs, + batch_size, + freeze_layers, + multimasks, + input_contact_spacing, + input_boundary_width, + ) + + logger.info(f"Training task {training_id} completed successfully") + return response except Exception as ex: training_instance.status = "FAILED" From 3a2558a3b1139e3fbdcc1e4367ec97bac0f6121d Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 12 Nov 2024 14:40:30 +0000 Subject: [PATCH 25/45] Added Multimasks Enabled for Yolo V1 --- backend/core/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index e3c6c74c..0a826404 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -301,7 +301,8 @@ def yolo_model_training( model_input_image_path = f"{base_path}/input" preprocess_output = f"/{base_path}/preprocessed" - + if model == "YOLO_V8_V1": + multimasks = True preprocess( input_path=model_input_image_path, output_path=preprocess_output, From 339e582059147d6ce25d43ca8dbe32901a2e259d Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 12 Nov 2024 14:53:29 +0000 Subject: [PATCH 26/45] Fix yolo copy dir problem for preprocessed output --- backend/core/tasks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 0a826404..47d7a8e1 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -365,9 +365,10 @@ def yolo_model_training( shutil.copyfile(output_model_path, os.path.join(output_path, "checkpoint.pt")) shutil.copytree(preprocess_output, os.path.join(output_path, "preprocessed")) os.makedirs(os.path.join(output_path,model),exist_ok=True) - shutil.copytree(preprocess_output, os.path.join(output_path,model, "images")) - shutil.copytree(preprocess_output, os.path.join(output_path,model, "labels")) - shutil.copytree(preprocess_output, os.path.join(output_path,model, "yolo_dataset.yaml")) + + shutil.copytree(os.path.join(yolo_data_dir,'images'), os.path.join(output_path,model, "images")) + shutil.copytree(os.path.join(yolo_data_dir,'labels'), os.path.join(output_path,model, "labels")) + shutil.copyfile(os.path.join(yolo_data_dir,'yolo_dataset.yaml'), os.path.join(output_path,model, "yolo_dataset.yaml")) graph_output_path = os.path.join( pathlib.Path(os.path.dirname(output_model_path)).parent, "iou_chart.png" From bb1f99c060ac42c38b7b8a8f8462ac4cbec8fc7d Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 12 Nov 2024 15:16:15 +0000 Subject: [PATCH 27/45] Dynamic epoch and batchsize for yolo v2 --- backend/core/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 47d7a8e1..b248320a 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -345,8 +345,8 @@ def yolo_model_training( output_model_path, final_accuracy = train_yolo_v2( data=f"{base_path}", weights=os.path.join(settings.YOLO_HOME, "yolov8s_v2-seg.pt"), - epochs=2, - batch_size=16, + epochs=epochs, + batch_size=batch_size, pc=2.0, output_path=yolo_data_dir, dataset_yaml_path=os.path.join(yolo_data_dir, "yolo_dataset.yaml"), From 1c9b65dcdbd3cfac37a58acc771ff8e65bcc2f91 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 12 Nov 2024 15:19:44 +0000 Subject: [PATCH 28/45] Added training publish limit --- backend/core/views.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/backend/core/views.py b/backend/core/views.py index a48cea75..686faa8a 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -744,15 +744,20 @@ def post(self, request, *args, **kwargs): def publish_training(request, training_id: int): """Publishes training for model""" training_instance = get_object_or_404(Training, id=training_id) + model_instance = get_object_or_404(Model, id=training_instance.model.id) if training_instance.status != "FINISHED": return Response("Training is not FINISHED", status=409) - if training_instance.accuracy < 70: - return Response( - "Can't publish the training since its accuracy is below 70%", status=403 - ) - - model_instance = get_object_or_404(Model, id=training_instance.model.id) + if model_instance.base_model == "RAMP": + if training_instance.accuracy < 70: + return Response( + "Can't publish the training since its accuracy is below 70%", status=403 + ) + else : ## Training publish limit for other model than ramp , TODO : Change this limit after testing for yolo + if training_instance.accuracy < 5: + return Response( + "Can't publish the training since its accuracy is below 5%", status=403 + ) # Check if the current user is the owner of the model if model_instance.user != request.user: From b3adcc332b1dcec88736ab74308a15526c263299 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 14 Nov 2024 14:27:41 +0000 Subject: [PATCH 29/45] refactor graph image --- backend/core/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/views.py b/backend/core/views.py index 686faa8a..fd4959f4 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -851,7 +851,7 @@ class TrainingWorkspaceDownloadView(APIView): def dispatch(self, request, *args, **kwargs): lookup_dir = kwargs.get("lookup_dir") - if lookup_dir.endswith("training_validation_sparse_categorical_accuracy.png"): + if lookup_dir.endswith("training_accuracy.png"): # bypass self.authentication_classes = [] self.permission_classes = [] From 18857a91e016378286684a09a196fac2b4c3db7a Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 21 Nov 2024 07:26:21 +0100 Subject: [PATCH 30/45] Enhance TrainingViewSet with filtering and searching capabilities --- backend/core/views.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/core/views.py b/backend/core/views.py index fd4959f4..1776d528 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -210,8 +210,15 @@ class TrainingViewSet( public_methods = ["GET"] queryset = Training.objects.all() http_method_names = ["get", "post", "delete"] + filter_backends = ( + DjangoFilterBackend, + filters.SearchFilter, + filters.OrderingFilter, + ) serializer_class = TrainingSerializer # connecting serializer filterset_fields = ["model", "status"] + ordering_fields = ["finished_at", "accuracy", "id", "model", "status"] + search_fields = ["description", "id"] def retrieve(self, request, *args, **kwargs): instance = self.get_object() @@ -753,7 +760,7 @@ def publish_training(request, training_id: int): return Response( "Can't publish the training since its accuracy is below 70%", status=403 ) - else : ## Training publish limit for other model than ramp , TODO : Change this limit after testing for yolo + else: ## Training publish limit for other model than ramp , TODO : Change this limit after testing for yolo if training_instance.accuracy < 5: return Response( "Can't publish the training since its accuracy is below 5%", status=403 From 4ef2ce85481bd1048b0b636a52476e9e98ea6eae Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 21 Nov 2024 10:20:15 +0100 Subject: [PATCH 31/45] Add configurable log line stream truncate value for task status logs --- backend/aiproject/settings.py | 3 +++ backend/core/views.py | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/aiproject/settings.py b/backend/aiproject/settings.py index 4d350bf3..c479b47f 100644 --- a/backend/aiproject/settings.py +++ b/backend/aiproject/settings.py @@ -226,4 +226,7 @@ ENABLE_PREDICTION_API = env("ENABLE_PREDICTION_API", default=False) +LOG_LINE_STREAM_TRUNCATE_VALUE = env("LOG_LINE_STREAM_TRUNCATE_VALUE", default=10) + + TEST_RUNNER = "tests.test_runners.NoDestroyTestRunner" diff --git a/backend/core/views.py b/backend/core/views.py index 1776d528..6e744ef2 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -557,9 +557,9 @@ def run_task_status(request, run_id: str): log_file = os.path.join(settings.LOG_PATH, f"run_{run_id}_log.txt") try: # read the last 10 lines of the log file - output = subprocess.check_output(["tail", "-n", "10", log_file]).decode( - "utf-8" - ) + output = subprocess.check_output( + ["tail", "-n", settings.LOG_LINE_STREAM_TRUNCATE_VALUE, log_file] + ).decode("utf-8") except Exception as e: output = str(e) result = { From 09ef0f8c3babeef0515fcd15b52c0d37abbd2982 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 25 Nov 2024 16:25:34 +0000 Subject: [PATCH 32/45] feat(yolo): added yolo inference support --- backend/core/tasks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index b248320a..d20fbaf7 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -363,6 +363,9 @@ def yolo_model_training( os.makedirs(output_path) shutil.copyfile(output_model_path, os.path.join(output_path, "checkpoint.pt")) + shutil.copyfile(os.path.join(os.path.dirname(output_model_path),'best.onnx'), os.path.join(output_path, "checkpoint.onnx")) + # shutil.copyfile(os.path.dirname(output_model_path,'checkpoint.tflite'), os.path.join(output_path, "checkpoint.tflite")) + shutil.copytree(preprocess_output, os.path.join(output_path, "preprocessed")) os.makedirs(os.path.join(output_path,model),exist_ok=True) From e8c3e6f97e50e7d7d84f8d68b7b87717f6520cc9 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 25 Nov 2024 16:29:53 +0000 Subject: [PATCH 33/45] ci(hot-fair-utilties): added requirement version upgrade --- backend/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index d7b4e8cc..ca8754e7 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,4 +1,4 @@ -r api-requirements.txt -hot-fair-utilities==1.3.0 +hot-fair-utilities==2.0.2 tflite-runtime==2.14.0 tippecanoe==2.45.0 \ No newline at end of file From 321dc9bbebfbc43503692a32cd4abfebdd43286e Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 25 Nov 2024 17:11:45 +0000 Subject: [PATCH 34/45] build(predictor): fixes fair predictor versions --- backend/api-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/api-requirements.txt b/backend/api-requirements.txt index e74dc0a7..91ed62b6 100644 --- a/backend/api-requirements.txt +++ b/backend/api-requirements.txt @@ -19,7 +19,7 @@ gpxpy==1.5.0 geojson2osm==0.0.1 osmconflator==0.0.11 orthogonalizer==0.0.4 -fairpredictor==0.0.26 +fairpredictor==0.0.36 rasterio==1.3.8 numpy<2.0.0 From da043eb99ee9a6172742107aa98e369586c68717 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 25 Nov 2024 18:19:41 +0100 Subject: [PATCH 35/45] chore(api-requirements): comment out orthogonalizer dependency --- backend/api-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/api-requirements.txt b/backend/api-requirements.txt index 91ed62b6..4d82f63c 100644 --- a/backend/api-requirements.txt +++ b/backend/api-requirements.txt @@ -18,7 +18,7 @@ validators==0.20.0 gpxpy==1.5.0 geojson2osm==0.0.1 osmconflator==0.0.11 -orthogonalizer==0.0.4 +# orthogonalizer==0.0.4 fairpredictor==0.0.36 rasterio==1.3.8 From f3b12283c60b4dcb6fa8438b7bfbe52b73084a18 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 25 Nov 2024 18:23:55 +0100 Subject: [PATCH 36/45] fix(view): restore orthogonalizer import for prediction API --- backend/core/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/views.py b/backend/core/views.py index 6e744ef2..fa01b834 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -32,7 +32,6 @@ from geojson2osm import geojson2osm from login.authentication import OsmAuthentication from login.permissions import IsAdminUser, IsOsmAuthenticated, IsStaffUser -from orthogonalizer import othogonalize_poly from osmconflator import conflate_geojson from rest_framework import decorators, filters, serializers, status, viewsets from rest_framework.decorators import api_view @@ -645,6 +644,7 @@ def post(self, request, *args, **kwargs): DEFAULT_TILE_SIZE = 256 if settings.ENABLE_PREDICTION_API: +from orthogonalizer import othogonalize_poly class PredictionView(APIView): authentication_classes = [OsmAuthentication] From 2c84cbb9e9f601f6482ee5f62bfa9229cd10cc65 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 25 Nov 2024 18:28:21 +0100 Subject: [PATCH 37/45] chore(docker): upgrade pip in Dockerfiles --- backend/Dockerfile | 1 + backend/Dockerfile_CPU | 3 +++ 2 files changed, 4 insertions(+) diff --git a/backend/Dockerfile b/backend/Dockerfile index 783c3b06..f2fdea49 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -9,6 +9,7 @@ RUN apt-get update && \ ENV CPLUS_INCLUDE_PATH=/usr/include/gdal ENV C_INCLUDE_PATH=/usr/include/gdal +RUN pip install --upgrade pip RUN pip install numpy==1.23.5 RUN pip install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==$(gdal-config --version) diff --git a/backend/Dockerfile_CPU b/backend/Dockerfile_CPU index d98f2a25..9a923041 100644 --- a/backend/Dockerfile_CPU +++ b/backend/Dockerfile_CPU @@ -10,6 +10,9 @@ RUN apt-get update && \ ENV CPLUS_INCLUDE_PATH=/usr/include/gdal ENV C_INCLUDE_PATH=/usr/include/gdal +RUN pip install --upgrade pip + + RUN pip install numpy==1.23.5 RUN pip install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==$(gdal-config --version) From e64216c0ac585076e1c082763fb499dfe2b3fe4e Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 25 Nov 2024 18:58:00 +0100 Subject: [PATCH 38/45] fix(docker): specify python3 for pip upgrade in Dockerfiles --- backend/Dockerfile | 2 +- backend/Dockerfile_CPU | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index f2fdea49..84f48cb0 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update && \ ENV CPLUS_INCLUDE_PATH=/usr/include/gdal ENV C_INCLUDE_PATH=/usr/include/gdal -RUN pip install --upgrade pip +RUN /usr/bin/python3 -m pip install --upgrade pip RUN pip install numpy==1.23.5 RUN pip install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==$(gdal-config --version) diff --git a/backend/Dockerfile_CPU b/backend/Dockerfile_CPU index 9a923041..2981ea3d 100644 --- a/backend/Dockerfile_CPU +++ b/backend/Dockerfile_CPU @@ -10,7 +10,7 @@ RUN apt-get update && \ ENV CPLUS_INCLUDE_PATH=/usr/include/gdal ENV C_INCLUDE_PATH=/usr/include/gdal -RUN pip install --upgrade pip +RUN /usr/bin/python3 -m pip install --upgrade pip RUN pip install numpy==1.23.5 From 4011806685e45211e6988844a8f48d644cbed813 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 25 Nov 2024 19:03:04 +0100 Subject: [PATCH 39/45] fix(docker): update pip installation command and upgrade fairpredictor version --- backend/Dockerfile | 3 +-- backend/Dockerfile_CPU | 5 +---- backend/pyproject.toml | 3 +-- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 84f48cb0..9f780f49 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -9,7 +9,6 @@ RUN apt-get update && \ ENV CPLUS_INCLUDE_PATH=/usr/include/gdal ENV C_INCLUDE_PATH=/usr/include/gdal -RUN /usr/bin/python3 -m pip install --upgrade pip RUN pip install numpy==1.23.5 RUN pip install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==$(gdal-config --version) @@ -28,7 +27,7 @@ RUN pip install setuptools==68.2.2 RUN pip install wheel==0.41.3 RUN pip install build==1.0.0 -RUN pip install -r requirements.txt +RUN pip install --upgrade pip && pip install -r requirements.txt # RUN pip install --use-deprecated=legacy-resolver -r requirements.txt COPY docker/ramp/solaris /tmp/solaris diff --git a/backend/Dockerfile_CPU b/backend/Dockerfile_CPU index 2981ea3d..45ac5652 100644 --- a/backend/Dockerfile_CPU +++ b/backend/Dockerfile_CPU @@ -10,9 +10,6 @@ RUN apt-get update && \ ENV CPLUS_INCLUDE_PATH=/usr/include/gdal ENV C_INCLUDE_PATH=/usr/include/gdal -RUN /usr/bin/python3 -m pip install --upgrade pip - - RUN pip install numpy==1.23.5 RUN pip install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==$(gdal-config --version) @@ -30,7 +27,7 @@ RUN pip install setuptools==68.2.2 RUN pip install wheel==0.41.3 RUN pip install build==1.0.0 -RUN pip install -r requirements.txt +RUN pip install --upgrade pip && pip install -r requirements.txt # RUN pip install --use-deprecated=legacy-resolver -r requirements.txt diff --git a/backend/pyproject.toml b/backend/pyproject.toml index e1eb0f5b..096cd654 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -25,8 +25,7 @@ dependencies = [ "gpxpy==1.5.0", "geojson2osm==0.0.1", "osmconflator==0.0.9", - "orthogonalizer==0.0.4", - "fairpredictor==0.0.26", + "fairpredictor==0.0.36", "tflite-runtime==2.14.0", "hot-fair-utilities==1.2.3", ] From e876342a15187b78b501dd516d3c45621f79666a Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 25 Nov 2024 19:13:13 +0100 Subject: [PATCH 40/45] fix(api-requirements): upgrade fairpredictor version to 0.0.37 --- backend/api-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/api-requirements.txt b/backend/api-requirements.txt index 4d82f63c..dabcdbdb 100644 --- a/backend/api-requirements.txt +++ b/backend/api-requirements.txt @@ -19,7 +19,7 @@ gpxpy==1.5.0 geojson2osm==0.0.1 osmconflator==0.0.11 # orthogonalizer==0.0.4 -fairpredictor==0.0.36 +fairpredictor==0.0.37 rasterio==1.3.8 numpy<2.0.0 From 3986a5b10ddefb45485bd4a6f297d113f6fde71a Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 25 Nov 2024 19:23:26 +0100 Subject: [PATCH 41/45] fix(requirements): upgrade hot-fair-utilities to version 2.0.4 --- backend/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index ca8754e7..3f4612e3 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,4 +1,4 @@ -r api-requirements.txt -hot-fair-utilities==2.0.2 +hot-fair-utilities==2.0.4 tflite-runtime==2.14.0 tippecanoe==2.45.0 \ No newline at end of file From 896513aca3362293d50360ec70118df5347cdc44 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 25 Nov 2024 19:33:03 +0100 Subject: [PATCH 42/45] fix(workflows): update Docker workflow to use ubuntu-24.04 --- .github/workflows/docker_publish_image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker_publish_image.yml b/.github/workflows/docker_publish_image.yml index 29fdc7f5..ffcbba92 100644 --- a/.github/workflows/docker_publish_image.yml +++ b/.github/workflows/docker_publish_image.yml @@ -19,7 +19,7 @@ env: jobs: build-and-push-api-image: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: read packages: write @@ -50,7 +50,7 @@ jobs: build-and-push-worker-image: needs: build-and-push-api-image - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: read packages: write From 175487f25f05141a6085e2ffe870127e2724f0b7 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 25 Nov 2024 21:59:45 +0100 Subject: [PATCH 43/45] fix(requirements): upgrade hot-fair-utilities to version 2.0.5 --- backend/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index 3f4612e3..2924b64b 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,4 +1,4 @@ -r api-requirements.txt -hot-fair-utilities==2.0.4 +hot-fair-utilities==2.0.5 tflite-runtime==2.14.0 tippecanoe==2.45.0 \ No newline at end of file From 7895852fc5f8c9139633383666f6705638011dcf Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 25 Nov 2024 22:08:37 +0100 Subject: [PATCH 44/45] fix(requirements): upgrade hot-fair-utilities to version 2.0.6 --- backend/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index 2924b64b..31e3b2b4 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,4 +1,4 @@ -r api-requirements.txt -hot-fair-utilities==2.0.5 +hot-fair-utilities==2.0.6 tflite-runtime==2.14.0 tippecanoe==2.45.0 \ No newline at end of file From e526d309adb206a1197d18e11d71e5ea3f66ef1a Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 25 Nov 2024 22:18:27 +0100 Subject: [PATCH 45/45] fix(docker): remove pip upgrade from Dockerfile installation commands --- backend/Dockerfile | 2 +- backend/Dockerfile_CPU | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 9f780f49..6f9e36fa 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -27,7 +27,7 @@ RUN pip install setuptools==68.2.2 RUN pip install wheel==0.41.3 RUN pip install build==1.0.0 -RUN pip install --upgrade pip && pip install -r requirements.txt +RUN pip install -r requirements.txt # RUN pip install --use-deprecated=legacy-resolver -r requirements.txt COPY docker/ramp/solaris /tmp/solaris diff --git a/backend/Dockerfile_CPU b/backend/Dockerfile_CPU index 45ac5652..d98f2a25 100644 --- a/backend/Dockerfile_CPU +++ b/backend/Dockerfile_CPU @@ -27,7 +27,7 @@ RUN pip install setuptools==68.2.2 RUN pip install wheel==0.41.3 RUN pip install build==1.0.0 -RUN pip install --upgrade pip && pip install -r requirements.txt +RUN pip install -r requirements.txt # RUN pip install --use-deprecated=legacy-resolver -r requirements.txt