WIP: Setup CI/CD

.gitignore

@@ -0,0 +1,2 @@
+.venv
+__pycache__

Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.12
+
+ARG PIP_EXTRA_INDEX_URL
+ARG ESG_FASTAPI_VERSION
+
+WORKDIR /app
+
+RUN export PIP_EXTRA_INDEX_URL="$PIP_EXTRA_INDEX_URL" && \
+    export ESGFHUB_VERSION="$ESG_FASTAPI_VERSION" && \
+    pip install \
+    esg_fastapi==$ESG_FASTAPI_VERSION \
+    --no-cache-dir \
+    --report -
+
+ENV PROMETHEUS_MULTIPROC_DIR /dev/shm
+
+CMD ["python", "-m", "esg_fastapi"]

Dockerfile.local

@@ -0,0 +1,9 @@
+FROM python:3.11
+
+WORKDIR /app
+COPY ../ .
+RUN pip install .
+
+ENV PROMETHEUS_MULTIPROC_DIR /dev/shm
+
+CMD ["python", "-m", "esg_fastapi"]

README.md

@@ -1,9 +1,20 @@
-# esg_fastapi
+# ESGF Search API
+This Project is developed as a speculative replacement for the existing Java based ESG Search API layer. The current implementation is based on FastAPI for an asynchronous framework.
 
+## Installation
+The primary installation target is Kubernetes via Helm, but since that requires a Docker container, you can also run that container locally via Docker or Podman. For example:
 ```bash
-pip install -r requirements
+podman run -it code.ornl.gov:4567/esgf/mirrors/esg_fastapi/esgf-esg-fastapi
 ```
 
+## Contributing
+To install and develop locally, use Poetry to create a virtual environment with the dependencies and configuration setup:
 ```bash
-uvicorn main:app --reload
-```
+poetry install
+```
+
+And start the server in `watch` mode with uvicorn:
+```bash
+uvicorn esg_fastapi.api --reload
+```
+While running in this mode, uvicorn will serve the API on http://localhost:8080 and watch for any changes to the source files. When detected, the server will reload the application automatically to include the new changes.

docker-compose/docker-compose.yaml

@@ -0,0 +1,101 @@
+version: "3"
+name: esg-fastapi
+
+volumes:
+  tempo-data:
+
+services:
+
+  esg-fastapi:
+    build:
+      context: ..
+      dockerfile: Dockerfile.local
+    network_mode: host
+    ports:
+    - "1337:1337"
+    logging:
+      driver: journald
+      options:
+        tag: esg-fastapi
+
+  pyroscope:
+    image: grafana/pyroscope
+    environment:
+      JAEGER_AGENT_HOST: localhost
+      JAEGER_SAMPLER_TYPE: const
+      JAEGER_SAMPLER_PARAM: 1
+    network_mode: host
+    command:
+    - "-server.grpc-listen-port=3030"
+    - "-config.file=/etc/pyroscope.yaml"
+    ports:
+    - '4040:4040'
+    volumes:
+    - ./pyroscope.yaml:/etc/pyroscope.yaml
+
+  tempo:
+    image: grafana/tempo
+    network_mode: host
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+    - ./tempo.yaml:/etc/tempo.yaml
+    - tempo-data:/var/tempo:rw,z
+    ports:
+    - "14268:14268"  # jaeger ingest
+    - "3200:3200"   # tempo
+    - "9095:9095" # tempo grpc
+    - "4317:4317"  # otlp grpc
+    - "4318:4318"  # otlp http
+    - "9411:9411"   # zipkin
+
+  prometheus:
+    image: prom/prometheus
+    network_mode: host
+    command:
+    - --config.file=/etc/prometheus.yaml
+    - --web.enable-remote-write-receiver
+    - --enable-feature=exemplar-storage
+    volumes:
+    - ./prometheus.yaml:/etc/prometheus.yaml
+    ports:
+    - "9090:9090"
+
+  grafana:
+    image: grafana/grafana
+    network_mode: host
+    volumes:
+    - ./grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+    environment:
+    - GF_AUTH_ANONYMOUS_ENABLED=true
+    - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+    - GF_AUTH_DISABLE_LOGIN_FORM=true
+    - GF_INSTALL_PLUGINS=pyroscope-panel
+    - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor traceQLStreaming metricsSummary traceToProfiles tracesEmbeddedFlameGraph traceToMetrics
+    ports:
+    - "3000:3000"
+
+  elasticsearch:
+    image: elasticsearch:8.13.0
+    network_mode: host
+    environment:
+    - bootstrap.memory_lock=false
+    - 'ES_JAVA_OPTS=-Xms512m -Xmx512m'
+    - discovery.type=single-node
+    - xpack.security.enabled=false
+    ports:
+    - '9200:9200'
+    - '9300:9300'
+
+  fluent-bit:
+    image: fluent/fluent-bit
+    network_mode: host
+    command:
+    - -c
+    - /fluent-bit/etc/fluent-bit.yaml
+    volumes:
+    - ./fluent-bit.yaml:/fluent-bit/etc/fluent-bit.yaml
+    - /etc/machine-id:/etc/machine-id:ro
+    - /var/log/journal:/var/log/journal
+    # depends_on:
+    # - elasticsearch
+    privileged: true

docker-compose/fluent-bit.yaml

@@ -0,0 +1,49 @@
+service:
+  daemon: Off
+  flush: 5
+  log_level: info
+
+pipeline:
+  inputs:
+  - name: systemd
+    Systemd_Filter: CONTAINER_TAG=esg-fastapi
+    tag: systemd
+
+    processors:
+      logs:
+      - name: content_modifier
+        action: extract
+        key: "MESSAGE"
+        context: body
+        pattern: 'trace_id=(?<trace_id>[^\s]+)'
+      - name: content_modifier
+        action: extract
+        key: "MESSAGE"
+        context: body
+        pattern: 'span_id=(?<span_id>[^\s]+)'
+      - name: content_modifier
+        action: extract
+        key: "MESSAGE"
+        context: body
+        pattern: 'resource\.service\.name=(?<resource_service_name>[^\s]+)'
+      - name: content_modifier
+        action: rename
+        key: resource_service_name
+        value: resource.service.name
+      - name: content_modifier
+        action: extract
+        key: "MESSAGE"
+        context: body
+        pattern: 'trace_sampled=(?<trace_sampled>True|False)'
+
+  outputs:
+  # - name: stdout
+  #   match: '*'
+  #   format: json_lines
+  - name: es
+    match: '*'
+    type: elasticsearch
+    host: localhost
+    port: 9200
+    logstash_format: On
+    suppress_type_name: On

docker-compose/grafana-datasources.yaml

@@ -0,0 +1,73 @@
+apiVersion: 1
+
+datasources:
+
+- name: Prometheus
+  type: prometheus
+  uid: prometheus
+  access: proxy
+  orgId: 1
+  url: http://localhost:9090
+  basicAuth: false
+  isDefault: false
+  version: 1
+  editable: false
+  jsonData:
+    httpMethod: GET
+    serviceMap:
+      datasourceUid: prometheus
+    tracesToProfiles:
+      customQuery: false
+      datasourceUid: "pyroscope"
+      profileTypeId: "process_cpu:cpu:nanoseconds:cpu:nanoseconds"
+
+- name: Tempo
+  type: tempo
+  access: proxy
+  orgId: 1
+  url: http://localhost:3200
+  basicAuth: false
+  isDefault: true
+  version: 1
+  editable: false
+  apiVersion: 1
+  uid: tempo
+  jsonData:
+    httpMethod: GET
+    nodeGraph:
+      enabled: true
+    serviceMap:
+      datasourceUid: prometheus
+    tracesToMetrics:
+      datasourceUid: prometheus
+    tracesToProfiles:
+      datasourceUid: pyroscope
+      profileTypeId: process_cpu:cpu:nanoseconds:cpu:nanoseconds
+    tracesToLogsV2:
+      datasourceUid: elasticsearch
+      filterByTraceID: true
+      tags: [{key: 'trace_id'}]
+
+- name: Pyroscope
+  type: grafana-pyroscope-datasource
+  access: proxy
+  orgId: 1
+  uid: pyroscope
+  url: http://localhost:4040
+
+- name: Elasticsearch
+  type: elasticsearch
+  access: proxy
+  orgId: 1
+  uid: elasticsearch
+  url: http://localhost:9200
+  jsonData:
+    index: '[logstash-]YYYY.MM.DD'
+    interval: Daily
+    timeField: '@timestamp'
+    logMessageField: MESSAGE
+    # logLevelField: fields.level
+    dataLinks:
+    - datasourceUid: tempo
+      field: trace_id
+      url: '$${__value.raw}'

docker-compose/prometheus.yaml

@@ -0,0 +1,8 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+- job_name: 'esg-fastapi'
+  static_configs:
+  - targets: ['localhost:1337']

docker-compose/pyroscope.yaml

@@ -0,0 +1,9 @@
+tracing:
+  enabled: true
+  profiling_enabled: true
+
+pyroscopedb:
+  max_block_duration: 5m
+
+self_profiling:
+  disable_push: true

docker-compose/tempo.yaml

@@ -0,0 +1,41 @@
+stream_over_http_enabled: true
+server:
+  http_listen_port: 3200
+  log_level: info
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+distributor:
+  receivers: # this configuration will listen on all ports and protocols that tempo is capable of.
+    jaeger: # the receives all come from the OpenTelemetry collector.  more configuration information can
+      protocols: # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
+        thrift_http: #
+        grpc: # for a production deployment you should only enable the receivers you need!
+        thrift_binary:
+        thrift_compact:
+    zipkin:
+    otlp:
+      protocols:
+        http:
+        grpc:
+    opencensus:
+
+ingester:
+  max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 1h # overall Tempo trace retention. set for demo purposes
+
+storage:
+  trace:
+    backend: local # backend configuration to use
+    wal:
+      path: /var/tempo/wal # where to store the the wal locally
+    local:
+      path: /var/tempo/blocks

esg_fastapi/__init__.py

@@ -0,0 +1,18 @@
+"""A REST API for Globus-based searches that mimics esg-search.
+
+This API is designed to be a thin wrapper around the `post_search` functionality of the Globus Search Index. It is intended to be bug-for-bug compatible with the ESG-Search API in both requests and responses to used by community tools that are based on the esg-search RESTful API, so that they do not need to change to be compatible with the new Globus indices.
+
+If you are designing a new project, you should look to use the globus-sdk directly.
+
+The stack consists of a FastAPI application that is configured to serve multiple versions of the API. Each version is represented by a separate FastAPI application, which is mounted as a sub-route of the main API at sub-routes like "/v1", "/v2", and so on.
+
+The main API also includes an observability endpoint at the "/observability" sub-route. This endpoint is used to expose OpenTelemetry generated by the API, such as health checks, request counts, response times, and error rates.
+"""
+
+from .configuration import settings
+
+__all__ = ["settings"]
+
+from .utils import print_loggers
+
+print_loggers()