Feature 18

.github/workflows/build_on_pull_request.yaml

@@ -20,58 +20,68 @@ jobs:
         build
         --user
     - name: Build a binary wheel and a source for drivers
-      run: python3 -m build ./drivers    
+      run: python3 -m build ./drivers
     - name: Set Docker image tag name
       run: echo "TAG=$(date +'%Y.%m.%d.%H.%M')" >> $GITHUB_ENV
+    - name: TAG ECHO
+      run: echo ${{ env.TAG }}
     - name: Login to Scaleway Container Registry
       uses: docker/login-action@v2
       with:
         username: nologin
         password: ${{ secrets.DOCKER_PASSWORD }}
         registry: ${{ vars.DOCKER_REGISTRY }}
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2     
+      uses: docker/setup-buildx-action@v2
     - name: Build and push drivers
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: ./drivers
         file: ./drivers/Dockerfile
         push: true
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         build-args: |
           REGISTRY=${{ vars.GEOKUBE_REGISTRY  }}
         tags: | 
           ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:${{ env.TAG }}  
-          ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:latest             
+          ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:latest
     - name: Build and push datastore component
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: ./datastore
         file: ./datastore/Dockerfile
         push: true
         build-args: |
           REGISTRY=${{ vars.DOCKER_REGISTRY  }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         tags: | 
           ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:${{ env.TAG }}  
-          ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:latest    
+          ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:latest
     - name: Build and push api component
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: ./api
         file: ./api/Dockerfile
         push: true
         build-args: |
           REGISTRY=${{ vars.DOCKER_REGISTRY  }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         tags: | 
           ${{ vars.DOCKER_REGISTRY }}/geolake-api:${{ env.TAG }}  
-          ${{ vars.DOCKER_REGISTRY }}/geolake-api:latest 
+          ${{ vars.DOCKER_REGISTRY }}/geolake-api:latest
     - name: Build and push executor component
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: ./executor
         file: ./executor/Dockerfile
         push: true
         build-args: |
           REGISTRY=${{ vars.DOCKER_REGISTRY  }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         tags: | 
           ${{ vars.DOCKER_REGISTRY }}/geolake-executor:${{ env.TAG }}  
-          ${{ vars.DOCKER_REGISTRY }}/geolake-executor:latest                           
+          ${{ vars.DOCKER_REGISTRY }}/geolake-executor:latest

.github/workflows/build_on_release.yaml

@@ -32,45 +32,54 @@ jobs:
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v2     
     - name: Build and push drivers
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: ./drivers
         file: ./drivers/Dockerfile
         push: true
         build-args: |
           REGISTRY=${{ vars.GEOKUBE_REGISTRY  }}
+          TAG=v0.2a6
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         tags: | 
           ${{ vars.GEOLAKE_REGISTRY }}/geolake-drivers:${{ env.RELEASE_TAG }}  
     - name: Build and push datastore component
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: ./datastore
         file: ./datastore/Dockerfile
         push: true
         build-args: |
           REGISTRY=${{ vars.GEOLAKE_REGISTRY  }}
           TAG=${{ env.RELEASE_TAG }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         tags: | 
           ${{ vars.GEOLAKE_REGISTRY }}/geolake-datastore:${{ env.RELEASE_TAG }}  
     - name: Build and push api component
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: ./api
         file: ./api/Dockerfile
         push: true
         build-args: |
           REGISTRY=${{ vars.GEOLAKE_REGISTRY  }}
           TAG=${{ env.RELEASE_TAG }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         tags: | 
           ${{ vars.GEOLAKE_REGISTRY }}/geolake-api:${{ env.RELEASE_TAG }}  
     - name: Build and push executor component
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: ./executor
         file: ./executor/Dockerfile
         push: true
         build-args: |
           REGISTRY=${{ vars.GEOLAKE_REGISTRY  }}
           TAG=${{ env.RELEASE_TAG }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         tags: | 
           ${{ vars.GEOLAKE_REGISTRY }}/geolake-executor:${{ env.RELEASE_TAG }}

.gitignore

@@ -112,5 +112,7 @@ venv.bak/
 _catalogs/
 _old/
 
-# Netcdf files
-*.nc
+.DS_Store
+db_init
+*.zarr
+*.nc

CITATION.cff

@@ -0,0 +1,51 @@
+# This CITATION.cff file was generated with cffinit.
+# Visit https://bit.ly/cffinit to generate yours today!
+
+cff-version: 1.2.0
+title: geolake
+message: >-
+  If you use this software, please cite it using the
+  metadata from this file.
+type: software
+authors:
+  - given-names: Marco
+    family-names: Mancini
+    orcid: 'https://orcid.org/0000-0002-9150-943X'
+  - given-names: Jakub
+    family-names: Walczak
+    orcid: 'https://orcid.org/0000-0002-5632-9484'
+  - given-names: Mirko
+    family-names: Stojiljković
+  - given-names: Valentina
+    family-names: Scardigno
+    orcid: 'https://orcid.org/0000-0002-0123-5368'
+identifiers:
+  - type: doi
+    value: 10.5281/zenodo.10598417
+repository-code: 'https://github.com/CMCC-Foundation/geolake'
+abstract: >+
+  geolake is an open source framework for management,
+  storage, and analytics of Earth Science data. geolake
+  implements the concept of a data lake as a central
+  location that holds a large amount of data in its native
+  and raw format. geolake does not impose any schema when
+  ingesting the data, however it provides a unified Data
+  Model and API for geoscientific datasets. The data is kept
+  in the original format and storage, and the in-memory data
+  structure is built on-the-fly for the processing analysis.
+
+  The system has been designed using a cloud-native
+  architecture, based on containerized microservices, that
+  facilitates the development, deployment and maintenance of
+  the system itself. It has been implemented by integrating
+  different open source frameworks, tools and libraries and
+  can be easily deployed using the Kubernetes platform and
+  related tools such as kubectl.
+
+keywords:
+  - python framework
+  - earth science
+  - data analytics
+license: Apache-2.0
+version: 0.1.0
+date-released: '2024-01-29'

README.md

@@ -1,3 +1,5 @@
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10598417.svg)](https://doi.org/10.5281/zenodo.10598417)
+
 # geolake
 
 ## Description

api/Dockerfile

@@ -1,9 +1,19 @@
 ARG REGISTRY=rg.fr-par.scw.cloud/geolake
 ARG TAG=latest
 FROM $REGISTRY/geolake-datastore:$TAG
+
+RUN apt update && apt install -y cron curl
+
 WORKDIR /app
 COPY requirements.txt /code/requirements.txt
 RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
 COPY app /app
 EXPOSE 80
+
+COPY ./healtcheck.* /opt/
+
+RUN chmod +x /opt/healtcheck.sh
+RUN crontab -u root /opt/healtcheck.cron
+
 CMD ["uvicorn", "app.main:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "80"]
+