diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..924ef88 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,11 @@ +# Set the default behavior, in case people don't have core.autocrlf set. +* text=auto + +# Known binary formats +*.png binary +*.jpg binary +*.ico binary +*.gif binary +*.ttf binary +*.woff binary +*.woff2 binary diff --git a/.github/actions/build-and-test-branch/action.yml b/.github/actions/build-and-test-branch/action.yml new file mode 100644 index 0000000..db4b898 --- /dev/null +++ b/.github/actions/build-and-test-branch/action.yml @@ -0,0 +1,120 @@ +name: 'Build and test branch' +description: 'Builds and tests a branch' +inputs: + branch-type: + description: 'String denoting either `target` or `feature` branch' + required: true + project-name: + description: 'String denoting the name of the project' + required: true + secrets: + description: 'Secrets from main.yml as JSON' +runs: + using: 'composite' + steps: + - name: Install Java, GDAL, and other system dependencies + run: | + sudo apt update + sudo apt-get install libxml2-dev libpq-dev openjdk-8-jdk libgdal-dev libxslt-dev + echo Postgres and ES dependencies installed + shell: bash + + - name: Set up Elasticsearch + uses: ankane/setup-elasticsearch@v1 + with: + elasticsearch-version: 8 + + - name: Install Python packages + run: | + python -m pip install --upgrade pip + pip install '.[dev]' + echo Python packages installed + shell: bash + + - name: Install Arches applications + uses: ./.github/actions/install-arches-applications + with: + secrets: ${{ inputs.secrets }} + + - name: Checkout into feature branch + if: inputs.branch-type == 'feature' + uses: actions/checkout@v4 + with: + repository: ${{ github.repository }} + ref: ${{ github.ref }} + path: . + + - name: Checkout into target branch + if: inputs.branch-type == 'target' + uses: actions/checkout@v4 + with: + repository: ${{ github.repository }} + ref: ${{ github.event.pull_request.base.ref }} + path: . + + - name: Install frontend dependencies + run: | + npm install + shell: bash + + - name: Webpack frontend files + run: | + npm run build_test + shell: bash + + - name: Check frontend formatting with prettier + run: | + npm run prettier:check + shell: bash + + - name: Check backend formatting with black + run: | + black . --check --exclude=node_modules + shell: bash + + - name: Check line endings + run: | + ! git ls-files --eol | grep 'w/crlf\|w/mixed' + shell: bash + + - name: Run frontend tests + run: | + npm run vitest + mv coverage/frontend/coverage.xml ${{ inputs.branch-type }}_branch_frontend_coverage.xml + shell: bash + + - name: Check for missing migrations + run: | + python manage.py makemigrations --check + shell: bash + + - name: Ensure previous Python coverage data is erased + run: | + coverage erase + shell: bash + + - name: Run Python unit tests + run: | + python -W default::DeprecationWarning -m coverage run manage.py test tests --settings="tests.test_settings" + shell: bash + + - name: Generate Python report coverage + run: | + coverage report + coverage json + mv coverage/python/coverage.json ${{ inputs.branch-type }}_branch_python_coverage.json + shell: bash + + - name: Upload frontend coverage report as artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.branch-type }}-branch-frontend-coverage-report + path: ${{ inputs.branch-type }}_branch_frontend_coverage.xml + overwrite: true + + - name: Upload Python coverage report as artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.branch-type }}-branch-python-coverage-report + path: ${{ inputs.branch-type }}_branch_python_coverage.json + overwrite: true diff --git a/.github/actions/install-arches-applications/action.yml b/.github/actions/install-arches-applications/action.yml new file mode 100644 index 0000000..703e33a --- /dev/null +++ b/.github/actions/install-arches-applications/action.yml @@ -0,0 +1,29 @@ +name: 'Install Arches Applications' +description: 'Manually edit this file to install all Arches Applications declared in settings.py, but not declared in `pyproject.toml`' +inputs: + secrets: + description: 'Secrets from main.yml as JSON' +runs: + using: 'composite' + steps: + + # Manually add any ARCHES_APPLICATIONS to this file if not already declared in `pyproject.toml`. + # Below is a template for adding an application in a private repository. + # Be sure to delete the `no-op step` if adding when updating this file. + + - name: No-op step to maintain workflow structure + run: echo "No-op step" + shell: bash + + # - name: Checkout ${my_arches_application_name} + # uses: actions/checkout@v4 + # with: + # repository: ${my_arches_application_repository}/${my_arches_application_name} + # token: ${{ fromJSON(inputs.secrets).${my_github_personal_access_token} }} + # path: ${my_arches_application_name} + + # - name: Install ${my_arches_application_name} + # run: | + # pip install ./${my_arches_application_name} + # echo ${my_arches_application_name} installed + # shell: bash diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..c12aac4 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,258 @@ +name: CI + +on: + # push: -- just run on PRs for now + pull_request: + workflow_dispatch: + +jobs: + build_feature_branch: + runs-on: ubuntu-latest + + services: + postgres: + image: postgis/postgis:13-3.0 + env: + POSTGRES_PASSWORD: postgis + POSTGRES_DB: disco + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + check-latest: true + + - name: Build and test branch + uses: ./.github/actions/build-and-test-branch + with: + secrets: ${{ toJSON(secrets) }} + project-name: 'disco' + branch-type: 'feature' + + build_target_branch: + runs-on: ubuntu-latest + + services: + postgres: + image: postgis/postgis:13-3.0 + env: + POSTGRES_PASSWORD: postgis + POSTGRES_DB: disco + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + check-latest: true + + - name: Build and test branch + uses: ./.github/actions/build-and-test-branch + with: + secrets: ${{ toJSON(secrets) }} + project-name: 'disco' + branch-type: 'target' + + check_frontend_coverage: + runs-on: ubuntu-latest + needs: [build_feature_branch, build_target_branch] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' # Use the latest available version + check-latest: true + + - name: Download feature branch frontend coverage report artifact + uses: actions/download-artifact@v4 + with: + name: feature-branch-frontend-coverage-report + path: . + + - name: Extract feature branch frontend coverage data + shell: pwsh + run: | + [xml]$xml = Get-Content feature_branch_frontend_coverage.xml + $metrics = $xml.coverage.project.metrics + + $statements = [double]$metrics.statements + $coveredstatements = [double]$metrics.coveredstatements + $conditionals = [double]$metrics.conditionals + $coveredconditionals = [double]$metrics.coveredconditionals + $methods = [double]$metrics.methods + $coveredmethods = [double]$metrics.coveredmethods + $elements = [double]$metrics.elements + $coveredelements = [double]$metrics.coveredelements + + $statement_coverage = 0.0 + $conditional_coverage = 0.0 + $method_coverage = 0.0 + $element_coverage = 0.0 + + if ($statements -gt 0) { + $statement_coverage = ($coveredstatements / $statements) * 100 + } + if ($conditionals -gt 0) { + $conditional_coverage = ($coveredconditionals / $conditionals) * 100 + } + if ($methods -gt 0) { + $method_coverage = ($coveredmethods / $methods) * 100 + } + if ($elements -gt 0) { + $element_coverage = ($coveredelements / $elements) * 100 + } + + $nonZeroCount = 0 + $totalCoverage = 0.0 + + if ($statements -gt 0) { $nonZeroCount++; $totalCoverage += $statement_coverage } + if ($conditionals -gt 0) { $nonZeroCount++; $totalCoverage += $conditional_coverage } + if ($methods -gt 0) { $nonZeroCount++; $totalCoverage += $method_coverage } + if ($elements -gt 0) { $nonZeroCount++; $totalCoverage += $element_coverage } + + $feature_branch_frontend_coverage = 0.0 + if ($nonZeroCount -gt 0) { + $feature_branch_frontend_coverage = $totalCoverage / $nonZeroCount + } + + Write-Output "feature_branch_frontend_coverage=$feature_branch_frontend_coverage" | Out-File -Append $env:GITHUB_ENV + + - name: Download target branch frontend coverage report artifact + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: target-branch-frontend-coverage-report + path: . + + - name: Check if target branch frontend coverage report artifact exists + run: | + if [ -f target_branch_frontend_coverage.xml ]; then + echo "target_branch_frontend_coverage_artifact_exists=true" >> $GITHUB_ENV + else + echo "Target branch coverage not found. Defaulting to 0% coverage." + echo "target_branch_frontend_coverage_artifact_exists=false" >> $GITHUB_ENV + fi + + - name: Extract target branch frontend coverage data + if: ${{ env.target_branch_frontend_coverage_artifact_exists == 'true' }} + shell: pwsh + run: | + [xml]$xml = Get-Content target_branch_frontend_coverage.xml + $metrics = $xml.coverage.project.metrics + + $statements = [double]$metrics.statements + $coveredstatements = [double]$metrics.coveredstatements + $conditionals = [double]$metrics.conditionals + $coveredconditionals = [double]$metrics.coveredconditionals + $methods = [double]$metrics.methods + $coveredmethods = [double]$metrics.coveredmethods + $elements = [double]$metrics.elements + $coveredelements = [double]$metrics.coveredelements + + $statement_coverage = 0.0 + $conditional_coverage = 0.0 + $method_coverage = 0.0 + $element_coverage = 0.0 + + if ($statements -gt 0) { + $statement_coverage = ($coveredstatements / $statements) * 100 + } + if ($conditionals -gt 0) { + $conditional_coverage = ($coveredconditionals / $conditionals) * 100 + } + if ($methods -gt 0) { + $method_coverage = ($coveredmethods / $methods) * 100 + } + if ($elements -gt 0) { + $element_coverage = ($coveredelements / $elements) * 100 + } + + $nonZeroCount = 0 + $totalCoverage = 0.0 + + if ($statements -gt 0) { $nonZeroCount++; $totalCoverage += $statement_coverage } + if ($conditionals -gt 0) { $nonZeroCount++; $totalCoverage += $conditional_coverage } + if ($methods -gt 0) { $nonZeroCount++; $totalCoverage += $method_coverage } + if ($elements -gt 0) { $nonZeroCount++; $totalCoverage += $element_coverage } + + $target_branch_frontend_coverage = 0.0 + if ($nonZeroCount -gt 0) { + $target_branch_frontend_coverage = $totalCoverage / $nonZeroCount + } + + Write-Output "target_branch_frontend_coverage=$target_branch_frontend_coverage" | Out-File -Append $env:GITHUB_ENV + + - name: Compare frontend feature coverage with target coverage + if: github.event_name == 'pull_request' + run: | + feature_branch_frontend_coverage=${feature_branch_frontend_coverage} + target_branch_frontend_coverage=${target_branch_frontend_coverage:-0.0} + + # Compare feature coverage with target coverage using floating-point comparison + if awk -v feature="$feature_branch_frontend_coverage" -v target="$target_branch_frontend_coverage" 'BEGIN { exit (feature < target) ? 0 : 1 }'; then + echo "Coverage decreased from $target_branch_frontend_coverage% to $feature_branch_frontend_coverage%. Please add or update tests to increase coverage." + exit 1 + else + echo "Feature branch coverage ($feature_branch_frontend_coverage%) >= Target branch coverage ($target_branch_frontend_coverage%)." + fi + + check_python_coverage: + runs-on: ubuntu-latest + needs: [build_feature_branch, build_target_branch] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' # Use the latest available version + check-latest: true + + - name: Download feature branch Python coverage report artifact + uses: actions/download-artifact@v4 + with: + name: feature-branch-python-coverage-report + path: . + + - name: Download target branch Python coverage report artifact + uses: actions/download-artifact@v4 + with: + name: target-branch-python-coverage-report + path: . + + - name: Compare Python feature coverage with target coverage + if: github.event_name == 'pull_request' + run: | + feature_branch_python_coverage=$(cat feature_branch_python_coverage.json | grep -o '"totals": {[^}]*' | grep -o '"percent_covered": [0-9.]*' | awk -F ': ' '{print $2}') + target_branch_python_coverage=$(cat target_branch_python_coverage.json | grep -o '"totals": {[^}]*' | grep -o '"percent_covered": [0-9.]*' | awk -F ': ' '{print $2}') + + # Compare feature coverage with target coverage using floating-point comparison + if awk -v feature="$feature_branch_python_coverage" -v target="$target_branch_python_coverage" 'BEGIN { exit (feature < target) ? 0 : 1 }'; then + echo "Coverage decreased from $target_branch_python_coverage% to $feature_branch_python_coverage%. Please add or update tests to increase coverage." + exit 1 + else + echo "Feature branch coverage ($feature_branch_python_coverage%) >= Target branch coverage ($target_branch_python_coverage%)." + fi diff --git a/.prettierrc b/.prettierrc index c4d8ac5..3baef1c 100644 --- a/.prettierrc +++ b/.prettierrc @@ -1,3 +1,4 @@ { - "singleAttributePerLine": true + "singleAttributePerLine": true, + "tabWidth": 4 } diff --git a/disco/apps.py b/disco/apps.py new file mode 100644 index 0000000..0b8fc94 --- /dev/null +++ b/disco/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig + + +class DiscoConfig(AppConfig): + name = "disco" + verbose_name = "Disco" + is_arches_application = True diff --git a/disco/settings.py b/disco/settings.py index de55019..1728023 100644 --- a/disco/settings.py +++ b/disco/settings.py @@ -130,7 +130,7 @@ "disco", ) -ARCHES_APPLICATIONS = ('arches_for_science',) +INSTALLED_APPS += ("arches.app",) MIDDLEWARE = [ "corsheaders.middleware.CorsMiddleware", @@ -149,17 +149,11 @@ # "silk.middleware.SilkyMiddleware", ] -STATICFILES_DIRS = build_staticfiles_dirs( - root_dir=ROOT_DIR, - app_root=APP_ROOT, - arches_applications=ARCHES_APPLICATIONS, -) +STATICFILES_DIRS = build_staticfiles_dirs(app_root=APP_ROOT) TEMPLATES = build_templates_config( - root_dir=ROOT_DIR, debug=DEBUG, app_root=APP_ROOT, - arches_applications=ARCHES_APPLICATIONS, context_processors=[ "django.contrib.auth.context_processors.auth", "django.template.context_processors.debug", @@ -488,13 +482,6 @@ pass # returns an output that can be read by NODEJS -if __name__ == "__main__": - transmit_webpack_django_config( - root_dir=ROOT_DIR, - app_root=APP_ROOT, - arches_applications=ARCHES_APPLICATIONS, - public_server_address=PUBLIC_SERVER_ADDRESS, - static_url=STATIC_URL, - webpack_development_server_port=WEBPACK_DEVELOPMENT_SERVER_PORT, - ) +if __name__ == "__main__": + transmit_webpack_django_config(**locals()) \ No newline at end of file diff --git a/disco/urls.py b/disco/urls.py index dc9a26b..bba30fc 100644 --- a/disco/urls.py +++ b/disco/urls.py @@ -12,7 +12,7 @@ # Only handle i18n routing in active project. This will still handle the routes provided by Arches core and Arches applications, # but handling i18n routes in multiple places causes application errors. -if settings.APP_NAME != "Arches" and settings.APP_NAME not in settings.ARCHES_APPLICATIONS: +if settings.ROOT_URLCONF == __name__: if settings.SHOW_LANGUAGE_SWITCH is True: urlpatterns = i18n_patterns(*urlpatterns) diff --git a/vitest.config.mts b/vitest.config.mts index 63a041c..34c23f9 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -9,7 +9,6 @@ const exclude = [ '**/cypress/**', '**/.{idea,git,cache,output,temp}/**', '**/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress,tsup,build}.config.*', - '**/src/declarations.d.ts', path.join(path.basename(__dirname), 'install', '**') ]; diff --git a/vitest.setup.mts b/vitest.setup.mts index 8a2d5cf..ca314b2 100644 --- a/vitest.setup.mts +++ b/vitest.setup.mts @@ -1,4 +1,5 @@ import { beforeAll, vi } from 'vitest'; +import '@/declarations.d.ts'; beforeAll(() => { diff --git a/webpack/webpack.common.js b/webpack/webpack.common.js index 5d58af0..e4104ee 100644 --- a/webpack/webpack.common.js +++ b/webpack/webpack.common.js @@ -316,15 +316,12 @@ module.exports = () => { loader: Path.join(PROJECT_RELATIVE_NODE_MODULES_PATH, 'vue-loader'), }, { - test: /\.m?js$/, + test: /\.mjs$/, include: /node_modules/, type: 'javascript/auto', - resolve: { - fullySpecified: false, - } }, { - test: /\.(js)$/, + test: /\.js$/, exclude: [/node_modules/, /load-component-dependencies/], loader: Path.join(PROJECT_RELATIVE_NODE_MODULES_PATH, 'babel-loader'), options: { @@ -453,26 +450,27 @@ module.exports = () => { } else { if (!isTestEnvironment) { - loaderContext.emitError(`Unable to fetch ${templatePath} from the Django server.`) + loaderContext.emitError(Error(`Unable to fetch ${templatePath} from the Django server.`)) } else { console.warn( '\x1b[31m%s\x1b[0m', // red `"${templatePath}" has failed to load! Test environment detected, falling back to un-rendered file.` ); - resp = { - text: () => ( - new Promise((resolve, _reject) => { - /* - if run in a test environment, failures will return a empty string which will - still allow the bundle to build. - */ - - resolve(isTestEnvironment ? '' : content); - }) - ) - }; } + + resp = { + text: () => ( + new Promise((resolve, _reject) => { + /* + if run in a test environment, failures will return a empty string which will + still allow the bundle to build. + */ + + resolve(isTestEnvironment ? '' : content); + }) + ) + }; } };