Skip to content

Commit

Permalink
Bugfix/sleepers tutorial hangs (#1777)
Browse files Browse the repository at this point in the history
add healthcheck in sidecar
adds TTL to log messages
removes cuda/nvidia container after test
give a bit more info when pulling the image
let celery handle exceptions
set redis as celery backend to get status update on webserver side
updated puppeteer to latest version 5.2.1 instead of 1.20.0 which should update the bundled chromium
force the sidecar node to run as a CPU node during e2e testing
  • Loading branch information
sanderegg authored Sep 9, 2020
1 parent 86a6a6c commit e9c5373
Show file tree
Hide file tree
Showing 34 changed files with 1,334 additions and 951 deletions.
2 changes: 1 addition & 1 deletion .env-devel
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ POSTGRES_PORT=5432
POSTGRES_USER=scu

RABBIT_HOST=rabbit
RABBIT_CHANNELS={"progress": "comp.backend.channels.progress", "log": "comp.backend.channels.log", "instrumentation": "comp.backend.channels.instrumentation", "celery": {"result_backend": "rpc://"}}
RABBIT_CHANNELS={"log": "comp.backend.channels.log", "instrumentation": "comp.backend.channels.instrumentation", "celery": {"result_backend": "redis://redis:6379/0"}}
RABBIT_PASSWORD=adminadmin
RABBIT_PORT=5672
RABBIT_USER=admin
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/ci-testing-deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
node: [10]
node: [14]
os: [ubuntu-20.04]
fail-fast: false
steps:
Expand Down Expand Up @@ -1014,7 +1014,7 @@ jobs:
strategy:
matrix:
python: [3.6]
node: [10]
node: [14]
os: [ubuntu-20.04]
fail-fast: false
steps:
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -155,3 +155,5 @@ prof/

# Copies
services/**/.codeclimate.yml
# WSL
.fake_hostname_file
7 changes: 7 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,13 @@ export SWARM_STACK_NAME_NO_HYPHEN = $(subst -,_,$(SWARM_STACK_NAME))
export DOCKER_IMAGE_TAG ?= latest
export DOCKER_REGISTRY ?= itisfoundation

# NOTE: this is only for WSL1 as /etc/hostname is not accessible there
ifeq ($(IS_WSL),WSL)
ETC_HOSTNAME = $(CURDIR)/.fake_hostname_file
export ETC_HOSTNAME
host := $(shell echo $$(hostname) > $(ETC_HOSTNAME))
endif


.PHONY: help

Expand Down
6 changes: 6 additions & 0 deletions ci/github/system-testing/e2e.bash
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ install_insecure_registry() {
echo DIRECTOR_REGISTRY_CACHING_TTL=0
# shorten time to sync services from director since registry comes later
echo CATALOG_BACKGROUND_TASK_REST_TIME=1
# ensure sidecars are started as CPU nodes
echo SIDECAR_FORCE_CPU_NODE=1
} >>.env

# prepare insecure registry access for docker engine
Expand Down Expand Up @@ -161,6 +163,10 @@ recover_artifacts() {
clean_up() {
echo "--------------- listing services running..."
docker service ls
echo "--------------- listing service details..."
docker service ps --no-trunc $(docker service ls --quiet)
echo "--------------- listing container details..."
docker container ps -a
echo "--------------- listing images available..."
docker images
echo "--------------- switching off..."
Expand Down
19 changes: 6 additions & 13 deletions packages/pytest-simcore/src/pytest_simcore/rabbit_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ def rabbit_config(docker_stack: Dict, devel_environ: Dict) -> Config:
host="127.0.0.1",
port=get_service_published_port("rabbit", devel_environ["RABBIT_PORT"]),
channels={
"progress": "progress_channel",
"log": "logs_channel",
"instrumentation": "instrumentation_channel",
"celery": {"result_backend": ""},
Expand Down Expand Up @@ -89,42 +88,36 @@ def channel_close_callback(exc: Optional[BaseException]):

@pytest.fixture(scope="function")
async def rabbit_exchange(
rabbit_config: Config, rabbit_channel: aio_pika.Channel,
) -> Tuple[aio_pika.Exchange, aio_pika.Exchange, aio_pika.Exchange]:
rabbit_config: Config,
rabbit_channel: aio_pika.Channel,
) -> Tuple[aio_pika.Exchange, aio_pika.Exchange]:

# declare log exchange
LOG_EXCHANGE_NAME: str = rabbit_config.channels["log"]
logs_exchange = await rabbit_channel.declare_exchange(
LOG_EXCHANGE_NAME, aio_pika.ExchangeType.FANOUT
)
assert logs_exchange
# declare progress exchange
PROGRESS_EXCHANGE_NAME: str = rabbit_config.channels["progress"]
progress_exchange = await rabbit_channel.declare_exchange(
PROGRESS_EXCHANGE_NAME, aio_pika.ExchangeType.FANOUT
)
assert progress_exchange
# declare instrumentation exchange
INSTRUMENTATION_EXCHANGE_NAME: str = rabbit_config.channels["instrumentation"]
instrumentation_exchange = await rabbit_channel.declare_exchange(
INSTRUMENTATION_EXCHANGE_NAME, aio_pika.ExchangeType.FANOUT
)
assert instrumentation_exchange
yield logs_exchange, progress_exchange, instrumentation_exchange
return logs_exchange, instrumentation_exchange


@pytest.fixture(scope="function")
async def rabbit_queue(
rabbit_channel: aio_pika.Channel,
rabbit_exchange: Tuple[aio_pika.Exchange, aio_pika.Exchange, aio_pika.Exchange],
rabbit_exchange: Tuple[aio_pika.Exchange, aio_pika.Exchange],
) -> aio_pika.Queue:
(logs_exchange, progress_exchange, instrumentation_exchange) = rabbit_exchange
(logs_exchange, instrumentation_exchange) = rabbit_exchange
# declare queue
queue = await rabbit_channel.declare_queue(exclusive=True)
assert queue
# Binding queue to exchange
await queue.bind(logs_exchange)
await queue.bind(progress_exchange)
await queue.bind(instrumentation_exchange)
yield queue

Expand Down
2 changes: 0 additions & 2 deletions packages/simcore-sdk/src/simcore_sdk/config/rabbit.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
"password": T.String(),
"channels": T.Dict(
{
"progress": T.String(),
"log": T.String(),
"instrumentation": T.String(),
T.Key(
Expand All @@ -45,7 +44,6 @@ class Config(BaseSettings):
host: str = "rabbit"
port: int = 5672
channels: Dict[str, Union[str, Dict]] = {
"progress": "comp.backend.channels.progress",
"log": "comp.backend.channels.log",
"instrumentation": "comp.backend.channels.instrumentation",
"celery": {"result_backend": "rpc://"},
Expand Down
12 changes: 12 additions & 0 deletions services/docker-compose-ops.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,18 @@ services:
networks:
- simcore_default

flower:
image: mher/flower:latest
init: true
restart: always
environment:
- CELERY_BROKER_URL=amqp://${RABBIT_USER}:${RABBIT_PASSWORD}@${RABBIT_HOST}:${RABBIT_PORT}
- FLOWER_PORT=5555
ports:
- "49555:5555"
networks:
- simcore_default

volumes:
minio_data: {}
portainer_data: {}
Expand Down
6 changes: 3 additions & 3 deletions services/docker-compose.devel.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ services:
- ./sidecar:/devel/services/sidecar
- ./storage/client-sdk:/devel/services/storage/client-sdk
- ../packages:/devel/packages
- /etc/hostname:/home/scu/hostname:ro
- ${ETC_HOSTNAME:-/etc/hostname}:/home/scu/hostname:ro
environment:
# force to start as cpu mode otherwise it will boot in gpu mode in development
- START_AS_MODE_CPU=1
Expand Down Expand Up @@ -82,7 +82,7 @@ services:
- ./sidecar:/devel/services/sidecar
- ./storage/client-sdk:/devel/services/storage/client-sdk
- ../packages:/devel/packages
- /etc/hostname:/home/scu/hostname:ro
- ${ETC_HOSTNAME:-/etc/hostname}:/home/scu/hostname:ro
ports:
- "3007:3000"
environment:
Expand Down Expand Up @@ -137,7 +137,7 @@ services:
- ./sidecar:/devel/services/sidecar
- ./storage/client-sdk:/devel/services/storage/client-sdk
- ../packages:/devel/packages
- /etc/hostname:/home/scu/hostname:ro
- ${ETC_HOSTNAME:-/etc/hostname}:/home/scu/hostname:ro
ports:
- "3008:3000"
environment:
Expand Down
2 changes: 1 addition & 1 deletion services/docker-compose.local.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ services:
rabbit:
ports:
- "5672"
- "15672"
- "15672:15672"
- "15692"

redis:
Expand Down
3 changes: 2 additions & 1 deletion services/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ services:
- output:/home/scu/output
- log:/home/scu/log
- /var/run/docker.sock:/var/run/docker.sock
- /etc/hostname:/home/scu/hostname:ro
- ${ETC_HOSTNAME:-/etc/hostname}:/home/scu/hostname:ro
environment:
- RABBIT_HOST=${RABBIT_HOST}
- RABBIT_PORT=${RABBIT_PORT}
Expand All @@ -172,6 +172,7 @@ services:
- REGISTRY_PW=${REGISTRY_PW}
- SWARM_STACK_NAME=${SWARM_STACK_NAME:-simcore}
- SIDECAR_LOGLEVEL=${LOG_LEVEL:-WARNING}
- START_AS_MODE_CPU=${SIDECAR_FORCE_CPU_NODE:-0}
- REDIS_HOST=${REDIS_HOST}
- REDIS_PORT=${REDIS_PORT}
depends_on:
Expand Down
20 changes: 14 additions & 6 deletions services/sidecar/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -112,12 +112,12 @@ COPY --from=cache --chown=scu:scu ${VIRTUAL_ENV} ${VIRTUAL_ENV}
# copy docker entrypoint and boot scripts
COPY --chown=scu:scu services/sidecar/docker services/sidecar/docker

# NOTE: when the sidecar gets an API, or maybe through rabbitMQ
# HEALTHCHECK --interval=30s \
# --timeout=20s \
# --start-period=30s \
# --retries=3 \
# CMD ["python3", "services/api-server/docker/healthcheck.py", "http://localhost:8000/"]
# we use the celery API
HEALTHCHECK --interval=60s \
--timeout=60s \
--start-period=30s \
--retries=3 \
CMD celery inspect ping -b amqp://${RABBIT_USER}:${RABBIT_PASSWORD}@${RABBIT_HOST}:${RABBIT_PORT} -d celery@${HOSTNAME}

ENTRYPOINT [ "/bin/sh", "services/sidecar/docker/entrypoint.sh" ]
CMD ["/bin/sh", "services/sidecar/docker/boot.sh"]
Expand All @@ -137,5 +137,13 @@ ENV SC_BUILD_TARGET=development

WORKDIR /devel
RUN chown -R scu:scu ${VIRTUAL_ENV}

# we use the celery API
HEALTHCHECK --interval=60s \
--timeout=60s \
--start-period=120s \
--retries=3 \
CMD celery inspect ping -b amqp://${RABBIT_USER}:${RABBIT_PASSWORD}@${RABBIT_HOST}:${RABBIT_PORT} -d celery@${HOSTNAME}

ENTRYPOINT [ "/bin/sh", "services/sidecar/docker/entrypoint.sh" ]
CMD ["/bin/sh", "services/sidecar/docker/boot.sh"]
3 changes: 1 addition & 2 deletions services/sidecar/requirements/_base.in
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@ aiopg
click
sqlalchemy>=1.3.3 # https://nvd.nist.gov/vuln/detail/CVE-2019-7164

celery
kombu
celery[redis]
networkx
packaging
pydantic
Expand Down
23 changes: 12 additions & 11 deletions services/sidecar/requirements/_base.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
#
# pip-compile --output-file=requirements/_base.txt requirements/_base.in
#
aio-pika==6.6.1 # via -r requirements/_base.in
aio-pika==6.7.0 # via -r requirements/_base.in
aiodocker==0.19.1 # via -r requirements/_base.in
aiofile==3.0.0 # via -r requirements/_base.in
aiofile==3.1.0 # via -r requirements/_base.in
aiofiles==0.5.0 # via -r requirements/_base.in
aiohttp==3.6.2 # via aiodocker
aiopg==1.0.0 # via -r requirements/_base.in
Expand All @@ -15,31 +15,32 @@ aioredlock==0.5.2 # via -r requirements/_base.in
aiormq==3.2.3 # via aio-pika
amqp==2.6.1 # via kombu
async-timeout==3.0.1 # via aiohttp, aioredis
attrs==19.3.0 # via aiohttp, aioredlock
attrs==20.2.0 # via aiohttp, aioredlock
billiard==3.6.3.0 # via celery
caio==0.5.3 # via aiofile
celery==4.4.7 # via -r requirements/_base.in
caio==0.6.1 # via aiofile
celery[redis]==4.4.7 # via -r requirements/_base.in
chardet==3.0.4 # via aiohttp
click==7.1.2 # via -r requirements/_base.in
dataclasses==0.7 # via pydantic
decorator==4.4.2 # via networkx
hiredis==1.1.0 # via aioredis
idna-ssl==1.1.0 # via aiohttp
idna==2.10 # via yarl
idna==2.10 # via idna-ssl, yarl
importlib-metadata==1.7.0 # via kombu
kombu==4.6.11 # via -r requirements/_base.in, celery
kombu==4.6.11 # via celery
multidict==4.7.6 # via aiohttp, yarl
networkx==2.4 # via -r requirements/_base.in
networkx==2.5 # via -r requirements/_base.in
packaging==20.4 # via -r requirements/_base.in
pamqp==2.3.0 # via aiormq
psycopg2-binary==2.8.5 # via aiopg
psycopg2-binary==2.8.6 # via aiopg
pydantic==1.6.1 # via -r requirements/_base.in
pyparsing==2.4.7 # via packaging
pytz==2020.1 # via celery
redis==3.5.3 # via celery
six==1.15.0 # via packaging, tenacity
sqlalchemy==1.3.18 # via -r requirements/_base.in
sqlalchemy==1.3.19 # via -r requirements/_base.in
tenacity==6.2.0 # via -r requirements/_base.in
typing-extensions==3.7.4.2 # via aiodocker, aiohttp, yarl
typing-extensions==3.7.4.3 # via aiodocker, aiohttp, yarl
urllib3==1.25.10 # via -r requirements/_base.in
vine==1.3.0 # via amqp, celery
yarl==1.5.1 # via aio-pika, aiohttp, aiormq
Expand Down
33 changes: 17 additions & 16 deletions services/sidecar/requirements/_test.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
#
# pip-compile --output-file=requirements/_test.txt requirements/_test.in
#
aio-pika==6.6.1 # via -r requirements/_base.txt
aio-pika==6.7.0 # via -r requirements/_base.txt
aiodocker==0.19.1 # via -r requirements/_base.txt
aiofile==3.0.0 # via -r requirements/_base.txt
aiofile==3.1.0 # via -r requirements/_base.txt
aiofiles==0.5.0 # via -r requirements/_base.txt
aiohttp==3.6.2 # via -r requirements/_base.txt, aiodocker, pytest-aiohttp
aiopg==1.0.0 # via -r requirements/_base.txt, -r requirements/_test.in
Expand All @@ -16,57 +16,58 @@ aiormq==3.2.3 # via -r requirements/_base.txt, aio-pika
amqp==2.6.1 # via -r requirements/_base.txt, kombu
astroid==2.4.2 # via pylint
async-timeout==3.0.1 # via -r requirements/_base.txt, aiohttp, aioredis
attrs==19.3.0 # via -r requirements/_base.txt, aiohttp, aioredlock, pytest
attrs==20.2.0 # via -r requirements/_base.txt, aiohttp, aioredlock, pytest
billiard==3.6.3.0 # via -r requirements/_base.txt, celery
caio==0.5.3 # via -r requirements/_base.txt, aiofile
celery==4.4.7 # via -r requirements/_base.txt
caio==0.6.1 # via -r requirements/_base.txt, aiofile
celery[redis]==4.4.7 # via -r requirements/_base.txt
certifi==2020.6.20 # via requests
chardet==3.0.4 # via -r requirements/_base.txt, aiohttp, requests
click==7.1.2 # via -r requirements/_base.txt
coverage==4.5.1 # via -r requirements/_test.in, coveralls, pytest-cov
coveralls==2.1.1 # via -r requirements/_test.in
coveralls==2.1.2 # via -r requirements/_test.in
dataclasses==0.7 # via -r requirements/_base.txt, pydantic
decorator==4.4.2 # via -r requirements/_base.txt, networkx
docker==4.2.2 # via -r requirements/_test.in
docker==4.3.1 # via -r requirements/_test.in
docopt==0.6.2 # via coveralls
hiredis==1.1.0 # via -r requirements/_base.txt, aioredis
idna-ssl==1.1.0 # via -r requirements/_base.txt, aiohttp
idna==2.10 # via -r requirements/_base.txt, idna-ssl, requests, yarl
importlib-metadata==1.7.0 # via -r requirements/_base.txt, kombu, pluggy, pytest
iniconfig==1.0.1 # via pytest
isort==4.3.21 # via pylint
isort==5.5.1 # via pylint
kombu==4.6.11 # via -r requirements/_base.txt, celery
lazy-object-proxy==1.4.3 # via astroid
mccabe==0.6.1 # via pylint
more-itertools==8.4.0 # via pytest
more-itertools==8.5.0 # via pytest
multidict==4.7.6 # via -r requirements/_base.txt, aiohttp, yarl
networkx==2.4 # via -r requirements/_base.txt
networkx==2.5 # via -r requirements/_base.txt
packaging==20.4 # via -r requirements/_base.txt, pytest, pytest-sugar
pamqp==2.3.0 # via -r requirements/_base.txt, aiormq
pluggy==0.13.1 # via pytest
psycopg2-binary==2.8.5 # via -r requirements/_base.txt, aiopg
psycopg2-binary==2.8.6 # via -r requirements/_base.txt, aiopg
ptvsd==4.3.2 # via -r requirements/_test.in
py==1.9.0 # via pytest
pydantic==1.6.1 # via -r requirements/_base.txt
pylint==2.5.3 # via -r requirements/_test.in
pylint==2.6.0 # via -r requirements/_test.in
pyparsing==2.4.7 # via -r requirements/_base.txt, packaging
pytest-aiohttp==0.3.0 # via -r requirements/_test.in
pytest-cov==2.10.0 # via -r requirements/_test.in
pytest-cov==2.10.1 # via -r requirements/_test.in
pytest-instafail==0.4.2 # via -r requirements/_test.in
pytest-lazy-fixture==0.6.3 # via -r requirements/_test.in
pytest-mock==3.2.0 # via -r requirements/_test.in
pytest-mock==3.3.1 # via -r requirements/_test.in
pytest-sugar==0.9.4 # via -r requirements/_test.in
pytest==6.0.1 # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-instafail, pytest-lazy-fixture, pytest-mock, pytest-sugar
python-dotenv==0.14.0 # via -r requirements/_test.in
pytz==2020.1 # via -r requirements/_base.txt, celery
redis==3.5.3 # via -r requirements/_base.txt, celery
requests==2.24.0 # via coveralls, docker
six==1.15.0 # via -r requirements/_base.txt, astroid, docker, packaging, tenacity, websocket-client
sqlalchemy==1.3.18 # via -r requirements/_base.txt
sqlalchemy==1.3.19 # via -r requirements/_base.txt
tenacity==6.2.0 # via -r requirements/_base.txt
termcolor==1.1.0 # via pytest-sugar
toml==0.10.1 # via pylint, pytest
typed-ast==1.4.1 # via astroid
typing-extensions==3.7.4.2 # via -r requirements/_base.txt, aiodocker, aiohttp, yarl
typing-extensions==3.7.4.3 # via -r requirements/_base.txt, aiodocker, aiohttp, yarl
urllib3==1.25.10 # via -r requirements/_base.txt, requests
vine==1.3.0 # via -r requirements/_base.txt, amqp, celery
websocket-client==0.57.0 # via docker
Expand Down
Loading

0 comments on commit e9c5373

Please sign in to comment.