Adding initial docker files (#1836)

* Adding initial docker files * impl review comments and add server config debug log * add server config debug log * Updates from #1844, add more configs to server * more changes from #1844 * make server configs parsable as args * make makefile accept args to pass to graphqlserver * fix python docker * add package and rust version as label and add dockerignore * add github workflow * deactivate storage * extract rust and package versions * impl review comments * fix pacakge name * upgrade rust * levelup rust ver * fix build --------- Co-authored-by: Ben Steer <[email protected]> Co-authored-by: Shivam Kapoor <[email protected]> Co-authored-by: Shivam <[email protected]>
Pometry · Nov 6, 2024 · 543435d · 543435d
1 parent 82607f4
commit 543435d
Show file tree

Hide file tree

Showing 20 changed files with 445 additions and 35 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,3 +1,5 @@
+pometry-storage-private
+.dockerignore
 target/
 .git/
 .env

diff --git a/.github/workflows/_release_dockerhub.yml b/.github/workflows/_release_dockerhub.yml
@@ -0,0 +1,75 @@
+name: _Release 5 - Publish Docker Images to Docker Hub
+on:
+  workflow_call:
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  publish-docker:
+    name: Build and Publish Docker Images
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v3
+
+      - name: Extract Package and Rust Versions
+        id: version_extraction
+        run: |
+          PACKAGE_VERSION=$(grep -m 1 '^version' Cargo.toml | sed 's/version = "\(.*\)"/\1/')
+          RUST_VERSION=$(grep -m 1 '^rust-version' Cargo.toml | sed 's/rust-version = "\(.*\)"/\1/')
+          echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> $GITHUB_ENV
+          echo "RUST_VERSION=$RUST_VERSION" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Deactivate Private Storage
+        run: |
+          chmod +x ./scripts/deactivate_private_storage.py
+          ./scripts/deactivate_private_storage.py
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set up QEMU for multi-platform builds
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build and push raphtory_base Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: docker/base/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ${{ secrets.DOCKERHUB_USERNAME }}/raphtory_base:${{ env.RUST_VERSION }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/raphtory_base:latest
+
+      - name: Build and push Python Docker image (pyraphtory)
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: docker/dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ${{ secrets.DOCKERHUB_USERNAME }}/pyraphtory:${{ env.PACKAGE_VERSION }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/pyraphtory:latest
+
+      - name: Build and push Rust Docker image (raphtory)
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ${{ secrets.DOCKERHUB_USERNAME }}/raphtory:${{ env.PACKAGE_VERSION }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/raphtory:latest
diff --git a/.github/workflows/manual_release_dockerhub.yml b/.github/workflows/manual_release_dockerhub.yml
@@ -0,0 +1,20 @@
+name: (Manual) Release Docker Hub
+on:
+  workflow_dispatch:
+    inputs:
+      base:
+        description: 'Name of branch to open PR against'
+        type: 'string'
+        default: 'main'
+      dry_run:
+        description: 'DRY RUN: If true, will not publish the release to PyPI/crates/Docker Hub but will release to GitHub'
+        type: boolean
+        default: false
+
+jobs:
+  call-release-docker-workflow:
+    name: _Release 5 - Publish Docker Images to Docker Hub
+    uses: ./.github/workflows/_release_docker.yml
+    with:
+      version: ${{ inputs.base }}
+    secrets: inherit
diff --git a/.github/workflows/release_auto.yml b/.github/workflows/release_auto.yml
@@ -32,3 +32,9 @@ jobs:
     with:
       base: ${{ inputs.base }}
     secrets: inherit
+  call-release-docker-workflow:
+    name: _Release 5 - Publish Docker Images to Docker Hub
+    uses: ./.github/workflows/_release_docker.yml
+    with:
+      version: ${{ inputs.base }}
+    secrets: inherit
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Dockerfile b/Dockerfile
@@ -1,25 +1,24 @@
-FROM rust:1.80.0 AS chef
+FROM rust:1.82.0 AS chef
 RUN cargo install cargo-chef --version 0.1.67
 WORKDIR /app
 
 FROM chef AS planner
 COPY . .
+RUN sed -i '/default-members/d' Cargo.toml
+RUN sed -i '/members = \[/,/\]/c\members = ["raphtory", "raphtory-graphql"]' Cargo.toml
 RUN cargo chef prepare  --recipe-path recipe.json
 
 FROM chef AS builder
 RUN apt-get update
 RUN apt-get install -y protobuf-compiler
 COPY --from=planner /app/recipe.json recipe.json
 RUN cargo chef cook --release --recipe-path recipe.json
-# RUN cargo chef cook --recipe-path recipe.json
 COPY . .
 RUN cargo build --release -p raphtory-graphql
-# RUN cargo build -p raphtory-graphql
 
-# FROM alpine:3.20.3
 FROM debian:bookworm-slim
 ENV PORT=1736
 COPY --from=builder /app/target/release/raphtory-graphql /raphtory-graphql
-# COPY --from=builder /app/target/debug/raphtory-graphql /usr/local/bin/raphtory-graphql
 WORKDIR /graphs
-CMD ["/raphtory-graphql"]
+
+ENTRYPOINT ["/raphtory-graphql"]
diff --git a/Makefile b/Makefile
@@ -63,4 +63,101 @@ debug-python: activate-storage
 	cd python && maturin develop --features=storage --extras=dev
 
 python-docs:
-	cd docs && make html
+	cd docs && make html
+
+WORKING_DIR ?= /tmp/graphs
+PORT ?= 1736
+
+PACKAGE_VERSION := $(shell grep -m 1 '^version' Cargo.toml | sed 's/version = "\(.*\)"/\1/')
+RUST_VERSION := $(shell grep -m 1 '^rust-version' Cargo.toml | sed 's/rust-version = "\(.*\)"/\1/')
+
+print-versions:
+	@echo "Package Version: $(PACKAGE_VERSION)"
+	@echo "Rust Version: $(RUST_VERSION)"
+
+BASE_IMAGE_NAME_AMD64 := pometry/raphtory_base:$(RUST_VERSION)-amd64
+BASE_IMAGE_NAME_ARM64 := pometry/raphtory_base:$(RUST_VERSION)-arm64
+IMAGE_NAME_AMD64 := pometry/raphtory:$(PACKAGE_VERSION)-rust-amd64
+IMAGE_NAME_ARM64 := pometry/raphtory:$(PACKAGE_VERSION)-rust-arm64
+PY_IMAGE_NAME_AMD64 := pometry/raphtory:$(PACKAGE_VERSION)-python-amd64
+PY_IMAGE_NAME_ARM64 := pometry/raphtory:$(PACKAGE_VERSION)-python-arm64
+
+docker-build-pyraphtory-base-amd64:
+	cd docker/base && docker build --platform linux/amd64 -t $(BASE_IMAGE_NAME_AMD64) .
+
+docker-build-pyraphtory-base-arm64:
+	cd docker/base && docker build --platform linux/arm64 -t $(BASE_IMAGE_NAME_ARM64) .
+
+docker-build-pyraphtory-amd64:
+	./scripts/deactivate_private_storage.py
+	docker build -f docker/dockerfile --build-arg BASE_IMAGE=$(BASE_IMAGE_NAME_AMD64) --platform linux/amd64 -t $(PY_IMAGE_NAME_AMD64) .
+
+docker-build-pyraphtory-arm64:
+	./scripts/deactivate_private_storage.py
+	docker build -f docker/dockerfile --build-arg BASE_IMAGE=$(BASE_IMAGE_NAME_ARM64) --platform linux/arm64 -t $(PY_IMAGE_NAME_ARM64) .
+
+docker-build-raphtory-amd64:
+	./scripts/deactivate_private_storage.py
+	docker build --platform linux/amd64 -t $(IMAGE_NAME_AMD64) .
+
+docker-build-raphtory-arm64:
+	./scripts/deactivate_private_storage.py
+	docker build --platform linux/arm64 -t $(IMAGE_NAME_ARM64) .
+
+# Docker run targets for pyraphtory
+docker-run-pyraphtory-amd64:
+	docker run --rm -p $(PORT):$(PORT) \
+		-v $(WORKING_DIR):/tmp/graphs \
+		$(PY_IMAGE_NAME_AMD64) \
+		$(if $(WORKING_DIR),--working-dir=$(WORKING_DIR)) \
+		$(if $(PORT),--port=$(PORT)) \
+		$(if $(CACHE_CAPACITY),--cache-capacity=$(CACHE_CAPACITY)) \
+		$(if $(CACHE_TTI_SECONDS),--cache-tti-seconds=$(CACHE_TTI_SECONDS)) \
+		$(if $(LOG_LEVEL),--log-level=$(LOG_LEVEL)) \
+		$(if $(TRACING),--tracing) \
+		$(if $(OTLP_AGENT_HOST),--otlp-agent-host=$(OTLP_AGENT_HOST)) \
+		$(if $(OTLP_AGENT_PORT),--otlp-agent-port=$(OTLP_AGENT_PORT)) \
+		$(if $(OTLP_TRACING_SERVICE_NAME),--otlp-tracing-service-name=$(OTLP_TRACING_SERVICE_NAME))
+
+docker-run-pyraphtory-arm64:
+	docker run --rm -p $(PORT):$(PORT) \
+		-v $(WORKING_DIR):/tmp/graphs \
+		$(PY_IMAGE_NAME_ARM64) \
+		$(if $(WORKING_DIR),--working-dir=$(WORKING_DIR)) \
+		$(if $(PORT),--port=$(PORT)) \
+		$(if $(CACHE_CAPACITY),--cache-capacity=$(CACHE_CAPACITY)) \
+		$(if $(CACHE_TTI_SECONDS),--cache-tti-seconds=$(CACHE_TTI_SECONDS)) \
+		$(if $(LOG_LEVEL),--log-level=$(LOG_LEVEL)) \
+		$(if $(TRACING),--tracing) \
+		$(if $(OTLP_AGENT_HOST),--otlp-agent-host=$(OTLP_AGENT_HOST)) \
+		$(if $(OTLP_AGENT_PORT),--otlp-agent-port=$(OTLP_AGENT_PORT)) \
+		$(if $(OTLP_TRACING_SERVICE_NAME),--otlp-tracing-service-name=$(OTLP_TRACING_SERVICE_NAME))
+
+# Docker run targets for raphtory
+docker-run-raphtory-amd64:
+	docker run --rm -p $(PORT):$(PORT) \
+		-v $(WORKING_DIR):/tmp/graphs \
+		$(IMAGE_NAME_AMD64) \
+		$(if $(WORKING_DIR),--working-dir=$(WORKING_DIR)) \
+		$(if $(PORT),--port=$(PORT)) \
+		$(if $(CACHE_CAPACITY),--cache-capacity=$(CACHE_CAPACITY)) \
+		$(if $(CACHE_TTI_SECONDS),--cache-tti-seconds=$(CACHE_TTI_SECONDS)) \
+		$(if $(LOG_LEVEL),--log-level=$(LOG_LEVEL)) \
+		$(if $(TRACING),--tracing) \
+		$(if $(OTLP_AGENT_HOST),--otlp-agent-host=$(OTLP_AGENT_HOST)) \
+		$(if $(OTLP_AGENT_PORT),--otlp-agent-port=$(OTLP_AGENT_PORT)) \
+		$(if $(OTLP_TRACING_SERVICE_NAME),--otlp-tracing-service-name=$(OTLP_TRACING_SERVICE_NAME))
+
+docker-run-raphtory-arm64:
+	docker run --rm -p $(PORT):$(PORT) \
+		-v $(WORKING_DIR):/tmp/graphs \
+		$(IMAGE_NAME_ARM64) \
+		$(if $(WORKING_DIR),--working-dir=$(WORKING_DIR)) \
+		$(if $(PORT),--port=$(PORT)) \
+		$(if $(CACHE_CAPACITY),--cache-capacity=$(CACHE_CAPACITY)) \
+		$(if $(CACHE_TTI_SECONDS),--cache-tti-seconds=$(CACHE_TTI_SECONDS)) \
+		$(if $(LOG_LEVEL),--log-level=$(LOG_LEVEL)) \
+		$(if $(TRACING),--tracing) \
+		$(if $(OTLP_AGENT_HOST),--otlp-agent-host=$(OTLP_AGENT_HOST)) \
+		$(if $(OTLP_AGENT_PORT),--otlp-agent-port=$(OTLP_AGENT_PORT)) \
+		$(if $(OTLP_TRACING_SERVICE_NAME),--otlp-tracing-service-name=$(OTLP_TRACING_SERVICE_NAME))
diff --git a/docker/.dockerignore b/docker/.dockerignore
@@ -0,0 +1,8 @@
+pometry-storage-private
+.dockerignore
+target/
+.git/
+.env
+.idea/
+.vscode/
+Dockerfile*
diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile
@@ -0,0 +1,29 @@
+FROM python:3.12.4-slim
+
+# Install packages & python base
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    python3 \
+    protobuf-compiler \
+    curl \
+    g++ \
+    git \
+    libssl-dev \
+    patchelf && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Create a virtual environment
+RUN python3 -m venv /opt/venv
+ENV VIRTUAL_ENV=/opt/venv
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# Install Rustup
+RUN curl --proto '=https' --tlsv1.3 https://sh.rustup.rs -sSf | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+RUN rustup toolchain install 1.82.0 && rustup default 1.82.0
+RUN cargo install --locked maturin
+
+WORKDIR /home/raphtory_server
+
+ENTRYPOINT [ "/bin/sh"]
diff --git a/docker/dockerfile b/docker/dockerfile
@@ -0,0 +1,27 @@
+#THIS SHOULD BE RUN FROM THE MAIN RAPHTORY DIR VIA MAKE
+# Stage 1: Build
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE} AS build
+
+WORKDIR /home/raphtory_server
+
+# Install custom raphtory and then delete the files
+COPY . /home/raphtory_server/raphtory
+RUN cd raphtory && rm -rf target && rm -rf pometry-storage-private
+RUN cd raphtory/python && maturin build -r
+RUN cd raphtory && pip install $(ls target/wheels/*.whl | head -n 1)
+RUN rm -rf raphtory
+
+RUN pip install python-dotenv
+
+# # Stage 2: Final
+FROM python:3.12.4-slim
+
+# Copy the virtual environment from the build stage
+COPY --from=build /opt/venv /opt/venv
+ENV VIRTUAL_ENV=/opt/venv
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+COPY docker/server.py /home/raphtory_server/server.py
+
+ENTRYPOINT ["python", "/home/raphtory_server/server.py"]
diff --git a/docker/server.py b/docker/server.py
@@ -0,0 +1,75 @@
+from raphtory import graphql
+from dotenv import load_dotenv
+import argparse
+
+# Load the .env file
+load_dotenv()
+
+parser = argparse.ArgumentParser(description="For passing the working_dir")
+parser.add_argument(
+    "--working-dir",
+    type=str,
+    default="graphs",
+    help="Path for the working directory of the raphtory server, defaults to 'graphs/'",
+)
+parser.add_argument(
+    "--port",
+    type=int,
+    default=1736,
+    help="Graphql server port, defaults to 1736",
+)
+parser.add_argument(
+    "--log-level",
+    type=str,
+    default="info",
+    help="Log level for the server, defaults to info",
+)
+parser.add_argument(
+    "--tracing",
+    type=bool,
+    default=False,
+    help="If tracing should be enabled or not, defaults to False",
+)
+parser.add_argument(
+    "--otlp-agent-host",
+    type=str,
+    default="localhost",
+    help="The address of the open telemetry collector, defaults to localhost",
+)
+parser.add_argument(
+    "--otlp-agent-port",
+    type=str,
+    default="4317",
+    help="The port of the open telemetry collector, default to 4317",
+)
+parser.add_argument(
+    "--otlp-tracing-service-name",
+    type=str,
+    default="Raphtory",
+    help="The name this service will be known by for open telemetry, default to Raphtory",
+)
+parser.add_argument(
+    "--cache-capacity",
+    type=int,
+    default=30,
+    help="The maximum amount of graphs to keep in memory at any given time, defaults to 30",
+)
+parser.add_argument(
+    "--cache-tti-seconds",
+    type=int,
+    default=900,
+    help="The amount of time a graph will be kept in memory before being dropped, defaults to 900 seconds",
+)
+args = parser.parse_args()
+
+server = graphql.GraphServer(
+    work_dir=args.working_dir,
+    tracing=args.tracing,
+    log_level=args.log_level,
+    otlp_agent_host=args.otlp_agent_host,
+    otlp_agent_port=args.otlp_agent_port,
+    otlp_tracing_service_name=args.otlp_tracing_service_name,
+    cache_capacity=args.cache_capacity,
+    cache_tti_seconds=args.cache_tti_seconds
+)
+server.run(port=args.port)