diff --git a/userportal/settings.py.dist b/userportal/settings.py.dist deleted file mode 100644 index 0f7b986..0000000 --- a/userportal/settings.py.dist +++ /dev/null @@ -1,434 +0,0 @@ -""" -Django settings for userportal project. - -Generated by 'django-admin startproject' using Django 3.1.7. - -For more information on this file, see -https://docs.djangoproject.com/en/3.1/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/3.1/ref/settings/ -""" - -from pathlib import Path -from django.utils.translation import gettext_lazy as _ -import os -import saml2 -import saml2.saml -import logging.config - -# Build paths inside the project like this: BASE_DIR / 'subdir'. -BASE_DIR = Path(__file__).resolve().parent.parent - - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'changeme' - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = ['portail.beluga.calculquebec.ca', 'localhost'] - -# Application definition - -INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'django.contrib.humanize', - 'djangosaml2', - 'watchman', - - 'pages', - 'slurm', - 'notes', - - 'jobstats', - 'accountstats', - 'cloudstats', - # 'quotas', - 'top', - 'usersummary', - - 'ccldap', - - # system modules - 'rest_framework', - 'rest_framework.authtoken', - 'bootstrap_pagination', -] - -MIDDLEWARE = [] -if DEBUG: - INSTALLED_APPS += ['debug_toolbar'] - MIDDLEWARE = ['debug_toolbar.middleware.DebugToolbarMiddleware'] - -MIDDLEWARE += [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.locale.LocaleMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.RemoteUserMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'djangosaml2.middleware.SamlSessionMiddleware', -] - -ROOT_URLCONF = 'userportal.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [os.path.join(BASE_DIR, 'templates')], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - 'django_settings_export.settings_export', - 'django.template.context_processors.request', - ], - }, - }, -] - -STATICFILES_DIRS = [ - os.path.join(BASE_DIR, 'static') -] - -WSGI_APPLICATION = 'userportal.wsgi.application' - - -# Database -# https://docs.djangoproject.com/en/3.1/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': 'userportaldev', - 'USER': 'userportaldev', - 'PASSWORD': 'changeme', - 'HOST': 'dbserver', - 'PORT': '3128', - 'OPTIONS': { - } - }, - 'slurm': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': 'slurmdb', - 'USER': 'slurm-stats', - 'PASSWORD': 'changeme', - 'HOST': 'dbserver', - 'PORT': '3128', - 'OPTIONS': { - }, - }, - 'ldap': { - 'ENGINE': 'ldapdb.backends.ldap', - 'NAME': 'ldaps://lproxy01', - 'USER': 'cn=ldapuser,dc=computecanada,dc=ca', - 'PASSWORD': 'changeme', - }, -} - -WATCHMAN_DATABASES = ['default', 'slurm'] - -DATABASE_ROUTERS = ['database_routers.dbrouters.DbRouter'] - -PROMETHEUS = { - 'url': 'https://thanos.dant.computecanada.ca', - 'headers': {'Authorization': 'Basic changeme_base64'}, - 'filter': "cluster='narval'", -} - -# Password validation -# https://docs.djangoproject.com/en/3.1/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, -] - - -# Internationalization -# https://docs.djangoproject.com/en/3.1/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - -LANGUAGES = [ - ('fr', _('French')), - ('en', _('English')), -] - -LOCALE_PATHS = ( - os.path.join(BASE_DIR, 'locale'), -) - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/3.1/howto/static-files/ - -STATIC_URL = '/static/' -STATIC_ROOT = '/opt/userportal/collected-static/' - -AUTHENTICATION_BACKENDS = [ - 'userportal.authentication.staffRemoteUserBackend', - 'django.contrib.auth.backends.ModelBackend', - 'userportal.authentication.staffSaml2Backend', -] - -LOGIN_REDIRECT_URL = '/' - -EXPORTER_INSTALLED = [ - 'slurm-job-exporter', - 'node_exporter', - 'redfish_exporter', - 'lfs_quota', - 'lustre_exporter', - 'slurm_exporter', -] - -EXPORTER_SAMPLING_RATE = { - 'slurm-job-exporter': 30, - 'node_exporter': 30, - 'redfish_exporter': 90, - 'lustre_exporter': 90, - 'slurm_exporter': 60, -} - -CLUSTER_NAME = 'narval' # used for slurm stats -STORAGE_NAME = 'narval-storage' -COMPUTE_NAME = 'narval' # narval-compute or narval-gpu -LUSTRE_FS_NAMES = ['lustre05', 'lustre06', 'lustre07'] -LOGINS = { - 'narval1': { - 'network_interface': 'eno34', - }, - 'narval2': { - 'network_interface': 'eno34', - }, - 'narval3': { - 'network_interface': 'eno34', - }, -} -DTNS = { - 'narval-dtn1': { - 'network_interface': 'bond0', - }, - 'narval-dtn2': { - 'network_interface': 'bond0', - }, - 'narval-dtn3': { - 'network_interface': 'bond0', - }, - 'narval-dtn4': { - 'network_interface': 'bond0', - }, -} - -# value used in the next variables, not showned in the UI, can be removed -HARDWARE_OVERHEAD = 1.10 # cables, racks, etc. -CURRENCY_CONVERSION = 1.25 # USD to CAD conversion - -# Can be set to None to disable the reporting -AMORTIZATION_YEARS = 5 # servers are amortized over 5 years -CPU_CORE_COST_PER_HOUR = 7500 / 64 / (365 * 24 * AMORTIZATION_YEARS) * HARDWARE_OVERHEAD -GPU_COST_PER_HOUR = 50000 / 4 / (365 * 24 * AMORTIZATION_YEARS) * HARDWARE_OVERHEAD - -ELECTRICITY_COST_PER_KWH = (4.158 / 100) # in $/kWh -COOLING_COST_PER_KWH = ELECTRICITY_COST_PER_KWH * 0.25 # not the exact value, but close enough -CO2_KG_PER_MWH = 0.5 # https://www.hydroquebec.com/data/developpement-durable/pdf/taux-emission-co2-approvisionnement-electricite-2020.pdf -ELECTRIC_CAR_RANGE_KM_PER_KWH = 1 / 0.151 # 151Wh/km https://ev-database.org/car/1555/Tesla-Model-3 - -CLOUD_CPU_CORE_COST_PER_HOUR = 2.448 / 64 * CURRENCY_CONVERSION # c6a.16xlarge = 64 cores, 128GB, half the memory of Narval -CLOUD_GPU_COST_PER_HOUR = 5.672 / 4 * CURRENCY_CONVERSION # g5.12xlarge = 48 cores, 4 GPUs, 192GB of ram, less than half the memory of Narval - -# Used to set reasonable thresholds for the mem usage in the "top compute" page -NORMAL_MEM_BY_CORE = 256/64 * 1024 * 1024 * 1024 # 256GB for 64 cores -NORMAL_MEM_BY_GPU = 512 / 4 * 1024 * 1024 * 1024 # 512GB for 4 GPUs -NORMAL_CORES_BY_GPU = 48 / 4 # 48 cores for 4 GPUs - -# Used to find the current quotas in prometheus -QUOTA_TYPES = { - 'home': ('/lustre05/home/', 'project'), - 'scratch': ('/lustre07/scratch/', 'project'), - 'project': ('/lustre06/project/', 'project'), -} - -SLURM_TRES = { - '1001=': 'NVIDIA A100-SXM4-40GB', -} - -CLUSTER_NAME_TITLE = 'Narval' -FAVICON = 'https://object-arbutus.cloud.computecanada.ca/userportal-public/narval.png' - -CLUSTER_INTRO = """ -

Narval is a general purpose cluster designed for a variety of workloads; it is located at the École de technologie supérieure in Montreal and is managed by Calcul Québec. The cluster is named in honour of the narwhal, a species of whale which has occasionally been observed in the Gulf of St. Lawrence.

- -

This cluster is composed of over 80720 cores and 636 GPUs, all nodes are connected at 100Gb/s.

- -

This portal is intended for our users, the menu on the left provides multiple tools to measure job performance and current cluster utilization

- -

Documentation

-""" - -SETTINGS_EXPORT = [ - 'CLUSTER_NAME_TITLE', - 'FAVICON', - 'CLUSTER_INTRO', - 'EXPORTER_INSTALLED', - 'INSTALLED_APPS', - 'LOGINS', - 'DTNS', - 'CLOUD_CPU_CORE_COST_PER_HOUR', - 'CLOUD_GPU_COST_PER_HOUR', - 'ELECTRICITY_COST_PER_KWH', - 'ELECTRIC_CAR_RANGE_KM_PER_KWH', - 'CO2_KG_PER_MWH', - 'AMORTIZATION_YEARS', - 'CPU_CORE_COST_PER_HOUR', - 'GPU_COST_PER_HOUR', -] - -INTERNAL_IPS = [ - '127.0.0.1', -] - -REST_FRAMEWORK = { - 'DEFAULT_PERMISSION_CLASSES': [ - 'rest_framework.permissions.IsAdminUser', - ], - 'DEFAULT_AUTHENTICATION_CLASSES': [ - 'rest_framework.authentication.TokenAuthentication', - 'rest_framework.authentication.SessionAuthentication', - ], - 'DEFAULT_RENDERER_CLASSES': ( - 'rest_framework.renderers.JSONRenderer', - 'rest_framework.renderers.BrowsableAPIRenderer', - 'rest_framework_datatables.renderers.DatatablesRenderer', - ), - 'DEFAULT_FILTER_BACKENDS': ( - 'rest_framework_datatables.filters.DatatablesFilterBackend', - ), - 'DEFAULT_PAGINATION_CLASS': 'rest_framework_datatables.pagination.DatatablesPageNumberPagination', - 'PAGE_SIZE': 100, -} - -CLOUD_ALLOCATIONS_FILE = '/var/www/userportal/projects-rac2022.yml' - -# SAML2 settings -SAML_SESSION_COOKIE_NAME = 'saml_session' -SESSION_COOKIE_SECURE = True -LOGIN_URL = '/saml2/login/' -SESSION_EXPIRE_AT_BROWSER_CLOSE = True -SAML_CREATE_UNKNOWN_USER = True - -SAML_CONFIG = { - 'debug': 1, - 'xmlsec_binary': '/usr/bin/xmlsec1', - 'entityid': 'http://simon.dev/saml2/metadata/', - - 'allow_unknown_attributes': True, - - 'service': { - 'sp': { - 'name': 'Test Userportal', - 'name_id_format': saml2.saml.NAMEID_FORMAT_PERSISTENT, - 'name_id_format_allow_create': True, - - 'endpoints': { - 'assertion_consumer_service': [ - ('http://localhost:8000/saml2/acs/', saml2.BINDING_HTTP_POST), - ], - }, - - 'signing_algorithm': saml2.xmldsig.SIG_RSA_SHA256, - 'digest_algorithm': saml2.xmldsig.DIGEST_SHA256, - - 'required_attributes': [ - 'surName', - 'givenName', - 'uid', - 'eduPersonAffiliation', - 'eduPersonPrincipalName', - 'displayName', - ], - - # When set to true, the SP will consume unsolicited SAML - # Responses, i.e. SAML Responses for which it has not sent - # a respective SAML Authentication Request. - 'allow_unsolicited': False, - }, - }, - - 'metadata': { - 'local': ['/opt/idp_metadata.xml'], # need to remove the POST SingleSignOnService from shibboleth metadata, only redirect seems to work - }, - - # Signing - 'key_file': '/opt/private.key', # private part - 'cert_file': '/opt/public.cert', # public part - - # Encryption - 'encryption_keypairs': [{ - 'key_file': '/opt/private.key', # private part - 'cert_file': '/opt/public.cert', # public part - }], -} - -# Logging Configuration - -# Clear prev config -LOGGING_CONFIG = None - -# Get loglevel from env -LOGLEVEL = os.getenv('DJANGO_LOGLEVEL', 'info').upper() - -logging.config.dictConfig({ - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'console': { - 'format': '%(asctime)s %(levelname)s [%(name)s:%(lineno)s] %(module)s %(process)d %(thread)d %(message)s', - }, - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'formatter': 'console', - }, - }, - 'loggers': { - '': { - 'level': LOGLEVEL, - 'handlers': ['console', ], - }, - }, -})