From a8350c298c73290e6b81c6bcd36bb0eaa48e416a Mon Sep 17 00:00:00 2001 From: Biyeun Buczyk Date: Mon, 16 Sep 2024 21:26:26 +0200 Subject: [PATCH 1/6] update hqDefine code check to not include ESM-formatted modules in migration metrics, have seperate ESM format module metric --- scripts/codechecks/hqDefine.sh | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/scripts/codechecks/hqDefine.sh b/scripts/codechecks/hqDefine.sh index 8811f24569a8..3a248f195e8a 100755 --- a/scripts/codechecks/hqDefine.sh +++ b/scripts/codechecks/hqDefine.sh @@ -8,6 +8,14 @@ function list-js() { find corehq custom -name '*.js' | grep -v '/_design/' | grep -v 'couchapps' | grep -v '/js/vellum/' } +function list-no-esm-js() { + list-js | xargs grep -L '^import.*;' +} + +function list-esm-js() { + list-js | xargs grep -l '^import.*;' +} + function list-html() { find corehq custom -name '*.html' | grep -v 'vellum' } @@ -19,13 +27,13 @@ function list-html-with-inline-scripts() { } function list-js-without-hqDefine() { - list-js | xargs grep -L 'hqDefine' + list-no-esm-js | xargs grep -L 'hqDefine' } # Partial indicator of RequireJS work left: how many js files don't yet use # the variation of hqDefine that specifies dependencies? function list-js-without-requirejs() { - list-js | xargs grep -L 'hqDefine.*\[' + list-no-esm-js | xargs grep -L 'hqDefine.*\[' } # The other indicator of RequireJS work left: how many HTML files still have script tags? @@ -43,9 +51,10 @@ function percent() { ## Main script command=${1:-""} -help="Pass list-script, list-hqdefine, list-requirejs, or list-requirejs-html to list the files that have yet to be migrated" +help="Pass list-script, list-hqdefine, list-requirejs, or list-requirejs-html to list the files that have yet to be migrated. list-esm to list ESM formatted files" jsTotalCount=$(echo $(list-js | wc -l)) +noEsmJsTotalCount=$(echo $(list-no-esm-js | wc -l)) htmlTotalCount=$(echo $(list-html | wc -l)) case $command in @@ -55,6 +64,11 @@ case $command in list-html-with-inline-scripts | sed 's/^/ /' ;; + "list-esm" ) + echo "These files use ESM syntax:" + list-esm-js | sed 's/^/ /' + ;; + "list-hqdefine" ) echo "The following files do not use hqDefine:" list-js-without-hqDefine | sed 's/^/ /' @@ -74,7 +88,7 @@ case $command in "static-analysis" ) withoutHqDefineCount=$(echo $(list-js-without-hqDefine | wc -l)) withoutRequireJsCount=$(echo $(list-js-without-requirejs | wc -l)) - echo "$withoutHqDefineCount $(($withoutRequireJsCount - $withoutHqDefineCount)) $(($jsTotalCount - $withoutRequireJsCount))" + echo "$withoutHqDefineCount $(($withoutRequireJsCount - $withoutHqDefineCount)) $(($noEsmJsTotalCount - $withoutRequireJsCount))" ;; "") @@ -86,14 +100,17 @@ case $command in echo "$(percent $unmigratedCount $htmlTotalCount) of HTML files are free of inline scripts" unmigratedCount=$(echo $(list-js-without-hqDefine | wc -l)) - echo "$(percent $unmigratedCount $jsTotalCount) of JS files use hqDefine" + echo "$(percent $unmigratedCount $noEsmJsTotalCount) of non-ESM JS files use hqDefine" unmigratedCount=$(echo $(list-js-without-requirejs | wc -l)) - echo "$(percent $unmigratedCount $jsTotalCount) of JS files specify their dependencies" + echo "$(percent $unmigratedCount $noEsmJsTotalCount) of non-ESM JS files specify their dependencies" unmigratedCount=$(echo $(list-html-with-external-scripts | wc -l)) echo "$(percent $unmigratedCount $htmlTotalCount) of HTML files are free of script tags" + unmigratedCount=$(echo $(list-no-esm-js | wc -l)) + echo "$(percent $unmigratedCount $jsTotalCount) of JS files use ESM format" + echo echo $help ;; From 7cfaf11150d11b0514c7a96e36bfc631d03557df Mon Sep 17 00:00:00 2001 From: Biyeun Buczyk Date: Mon, 16 Sep 2024 21:34:10 +0200 Subject: [PATCH 2/6] update the JavaScript Guide to contian information re: Webpack --- docs/js-guide/README.rst | 11 +- docs/js-guide/amd-to-esm.rst | 206 ++++++++++++ docs/js-guide/code-organization.rst | 17 +- docs/js-guide/dependencies.rst | 427 ++++++++++++++++--------- docs/js-guide/migrating.rst | 145 +++++---- docs/js-guide/module-history.rst | 54 ++-- docs/js-guide/requirejs-to-webpack.rst | 214 +++++++++++++ docs/js-guide/static-files.rst | 125 +++++++- docs/js-guide/testing.rst | 2 +- docs/translations.rst | 6 +- 10 files changed, 952 insertions(+), 255 deletions(-) create mode 100644 docs/js-guide/amd-to-esm.rst create mode 100644 docs/js-guide/requirejs-to-webpack.rst diff --git a/docs/js-guide/README.rst b/docs/js-guide/README.rst index 743c2a2d4e92..d772c42fcb7f 100644 --- a/docs/js-guide/README.rst +++ b/docs/js-guide/README.rst @@ -15,6 +15,7 @@ Table of contents :maxdepth: 1 code-organization + static-files .. toctree:: :caption: Dependencies @@ -22,17 +23,23 @@ Table of contents dependencies module-history - migrating libraries external-packages +.. toctree:: + :caption: Migrations + :maxdepth: 1 + + migrating + requirejs-to-webpack + amd-to-esm + .. toctree:: :caption: Best practices :maxdepth: 1 integration-patterns security - static-files inheritance code-review diff --git a/docs/js-guide/amd-to-esm.rst b/docs/js-guide/amd-to-esm.rst new file mode 100644 index 000000000000..236c96a7e3d1 --- /dev/null +++ b/docs/js-guide/amd-to-esm.rst @@ -0,0 +1,206 @@ +Updating Module Syntax from AMD to ESM +====================================== + +Most entry points for legacy modules that have recently been migrated from RequireJS to +Webpack as part of the `RequireJS to Webpack Migration +`__ +are eligible for this update. + +See the `Historical Background on Module Patterns +`__ +for a more detailed discussion of module types. As a quick refresher, here are some definitions: + +AMD (Asynchronous Module Definition) + The legacy module type used for older JavaScript modules on HQ. + This was the only module type compatible with RequireJS, our first JavaScript bundler. + It is still needed as a format for modules required by No-Bundler pages. + +ESM (ES Modules) + The newest module type with updated powerful import and export syntax. This is the module + format that you will see referenced by documentation in modern javascript frameworks. + +The different types of modules you will encounter are: + +Entry Point Modules + Modules that are included directly on a page using a bundler template tag, like + ``webpack_main``. These are the modules that the bundler (Webpack) uses to build + a dependency graph so that it knows what bundle of javascript dependencies and + page-specific code is needed to render that page / entry point. + +Dependency Modules + These are modules that are never referenced in a bundler template tag and are only + in the list of dependencies for other modules. Often these modules are used as utility modules + or a way to organize JavaScript for a page that is very front-end heavy. + + +Step 1: Determine if the Module is Eligible for a Syntax Update +--------------------------------------------------------------- + +The HQ AMD-style module will look something like: + +:: + + hqDefine('hqwebapp/js/my_module', [ + 'jquery', + 'knockout', + 'underscore', + 'hqwebapp/js/initial_page_data', + 'hqwebapp/js/assert_properties', + 'hqwebapp/js/bootstrap5/knockout_bindings.ko', + 'commcarehq', + ], function ( + $, + ko, + _, + initialPageData, + assertProperties + ) { + ... + }); + + +Entry Points +~~~~~~~~~~~~ + +If this module is a webpack entry point, then it is eligible for an update. In the example above, you would find +``hqwebapp/js/my_module`` used on a page with the following: + +:: + + {% webpack_main "hqwebapp/js/my_module %} + +The entry point can also be specified with ``webpack_main_b3`` if the module is part of the Bootstrap 3 build +of Webpack. + +If this module is inside a ``requirejs_main`` or ``requirejs_main_b5`` tag, then it is NOT eligible for an update. +Instead, please first +`migrate this module from RequireJS to Webpack `__ + +Dependency Modules +~~~~~~~~~~~~~~~~~~ + +If this module is a dependency of any modules that are ``requirejs_main`` entry points, then this module is not +eligible for migration. If a module's syntax is updated when it's still required by RequireJS modules, then +it will result in a RequireJS build failure on deploy. + +If this module is referenced by any ``hqImport`` calls (for instance ``hqImport('hqwebapp/js/my_module')``), +then this module is NOT yet eligible, and must continue using the older AMD-style syntax until +the ``hqImport`` statements are no longer needed. See the +`JS Bundler Migration Guide `__ for +how to proceed in this case. + +Slightly Different Syntax +~~~~~~~~~~~~~~~~~~~~~~~~~ + +If the AMD-style module looks a bit different than the syntax above--for instance, the list of dependencies are missing or +``hqImport`` and/or global variables can be found in the main body of the module--then this module must be +`migrated to use a JS Bundler `__. + + +Step 2: Update Surrounding Module Structure +------------------------------------------- + +ESM no longer needs to define the module name within the module itself. Instead, Webpack (our bundler) is configured +to know how to reference this module by its filename and relative path within an application. + +You can start this by changing: + +:: + + hqDefine('hqwebapp/js/my_module', [ + 'jquery', + 'knockout', + 'underscore', + 'hqwebapp/js/initial_page_data', + 'hqwebapp/js/assert_properties', + 'hqwebapp/js/bootstrap5/knockout_bindings.ko', + 'commcarehq', + ], function ( + $, + ko, + _, + initialPageData, + assertProperties + ) { + ... + }); + +to + +:: + + [ + 'jquery', + 'knockout', + 'underscore', + 'hqwebapp/js/initial_page_data', + 'hqwebapp/js/assert_properties', + 'hqwebapp/js/bootstrap5/knockout_bindings.ko', + 'commcarehq', + ] ( + $, + ko, + _, + initialPageData, + assertProperties + ) + // additionally, the indentation for the module-specific code can be updated + ... + +Step 3: Update Dependency Imports +--------------------------------- + +Common ``yarn`` dependencies can now be referenced by their NPM name (or the alias defined in +``webpack/webpack.common.js``. + +The same can be done with named internal dependencies, or referenced internal dependencies. + +The final module dependency structure will look something like: + +:: + + import "commcarehq"; // Note: moved to top + + // named yarn/npm dependencies + import $ from "jquery"; + import ko from "knockout"; + import _ from "underscore"; + + // named internal dependencies: + import initialPageData from "hqwebapp/js/initial_page_data"; + import assertProperties from "hqwebapp/js/assert_properties"; + + // referenced internal dependencies: + import "hqwebapp/js/bootstrap3/knockout_bindings.ko"; + + // module specific code... + ... + +Note that ``import "commcarehq";`` has been moved to the top of the file. The ordering is +for consistency purposes, but it's important that either ``import "commcarehq";`` or +``import "commcarehq_b3";`` (for Bootstrap 3 / ``webpack_main_b3``) is present in the list +of imports for Webpack Entry Point modules. + +Remember, an Entry Point is any module that is included directly on a page using the +``webpack_main`` or ``webpack_main_b3`` template tags. + +Modules that are not entry points are not required to have this import. If you are updating the +syntax of a dependency (non-entry point) module, do not worry about including this import if +it is not already present. + + +Step 4: Other Code Updates +-------------------------- + +If this module is an entry point, then the rest of the module-specific code can remain as is, +with the indentation level updated. However, some entry points are also dependencies of other +entry points. If that's the case, proceed to the next part. + +If this module is a dependency module, meaning it is referenced by other modules, +then the ``return`` line at the end of the module should follow the appropriate ``export`` syntax +needed by the modules that depend on this module. + +The most likely change is to replace ``return`` with ``export`` and leave everything else as is. +Otherwise, see the +`export documentation `__ +for details and inspiration in case you want to do some additional refactoring. diff --git a/docs/js-guide/code-organization.rst b/docs/js-guide/code-organization.rst index a46bf0370ef9..5991963eee37 100644 --- a/docs/js-guide/code-organization.rst +++ b/docs/js-guide/code-organization.rst @@ -1,8 +1,9 @@ Static Files Organization ------------------------- -All\* JavaScript code should be in a .js file and encapsulated as a -module using ``hqDefine``. +All JavaScript code should be in a .js file and encapsulated as a +module either using the ES Module syntax or modified-AMD syntax in +legacy code using using ``hqDefine``. JavaScript files belong in the ``static`` directory of a Django app, which we structure as follows: @@ -15,15 +16,15 @@ which we structure as follows: font/ images/ js/ <= JavaScript - less/ + scss/ lib/ <= Third-party code: This should be rare, since most third-party code should be coming from yarn spec/ <= JavaScript tests ... <= May contain other directories for data files, i.e., `json/` templates/myapp/ mytemplate.html -\* There are a few places we do intentionally use script blocks, such as -configuring less.js in CommCare HQ’s main template, -``hqwebapp/base.html``. These are places where there are just a few -lines of code that are truly independent of the rest of the site’s -JavaScript. They are rare. +To develop with javascript locally, make sure you run ``yarn dev`` and +restart ``yarn dev`` whenever you add a new Webpack Entry Point. + +Please review the next section for a more detailed discussion of Static Files +and the use of JavaScript bundlers, like Webpack. diff --git a/docs/js-guide/dependencies.rst b/docs/js-guide/dependencies.rst index ecc32325a621..570f3fc81d9b 100644 --- a/docs/js-guide/dependencies.rst +++ b/docs/js-guide/dependencies.rst @@ -1,72 +1,145 @@ Managing Dependencies ===================== -HQ’s JavaScript is being gradually migrated from a legacy, unstructured -coding style the relies on the ordering of script tags to instead use -RequireJS for dependency management. This means that dependencies are -managed differently depending on which area of the code you’re working -in. This page is a developer’s guide to understanding which area you’re -working in and what that means for your code. +Front-end dependencies are managed using ``yarn`` and are defined in ``package.json`` at the +root of the ``commcare-hq`` repository. -My python tests are failing because of javascript -------------------------------------------------- -`TestRequireJS -`__ -reads all of our javascript files, checking for common errors. +Most JavaScript on HQ is included on a page via a JavaScript bundle. +These bundles are created by a JavaScript bundler. The bundler is given a +list of "entry points" (or pages), builds a dependency graph of modules to determine what +code is needed for a page, and combines related code into bundles. +These bundles are split along ``vendor`` (npm modules), +``common`` (all of hq), and application (like ``hqwebapp`` or ``domain``). -These tests are naive. They don't parse JavaScript, they just run regexes based on expected coding patterns. -They use `this method <#how-do-i-know-whether-or-not-im-working-with-requirejs>`__ to determine if a file is -using RequireJS. This is one reason not to add dependency lists in areas of HQ that don't yet use RequireJS. +By bundling code, we can make fewer round-trip requests to fetch all of a page's JavaScript. +Additionally, the bundler minifies each bundle to reduce its overall size. You can learn +more about bundlers in `the Static Files Overview +`__ -**test_requirejs_disallows_hqimport** +HQ is transitioning from RequireJS (our original JavaScript bundler, which is no longer maintained) +to Webpack. Portions of this documentation that reference RequireJS will remain until RequireJS +has been fully removed from the codebase. See the `RequireJS to Webpack Migration Guide +`__ +for more information about this migration. -``hqImport`` only works in non-RequireJS contexts. In RequireJS files, dependencies should be included in the -module's ``hqDefine`` call, as described `here <#how-do-i-know-whether-or-not-im-working-with-requirejs>`__. +A few areas of our codebase are also undergoing a migration toward using a bundler. +You can read more about this migration in the `JS Bundler Migration Guide +`__ -Occasionally, this does not work due to a circular dependency. This will manifest as the module being undefined. -`hqRequire `__ -exists for this purpose, to require the necessary module at the point where it’s used. ``hqRequire`` defines -a new module, which can be fragile, so limit the code using it. As in python, best practice is to include -dependencies at the module level, at the top of the file. +Before adopting a bundler, HQ's javascript followed a more legacy, unstructured coding style +that relied on ordering script tags on a page. This approach required more individual +requests to fetch each dependency separately. "Bundles" were created manually by grouping +script tags inside the ``compress`` template tag managed by Django Compressor. +If you are modifying code inside a section that still follows this structure, the way you +import external modules/dependencies for use in that page's code will differ from the module +approach on pages using either Webpack or RequireJS. We will address this legacy approach +at the end of this chapter. -**test_files_match_modules** -RequireJS requires that a module's name is the same as the file containing it. Rename your module. +How do I create a new page with JavaScript? +------------------------------------------- -My deploy is failing because of javascript ------------------------------------------- +New code should be written using the ES Module (ESM) format and bundled using Webpack. This approach +is oriented around a single "entry point" per page (with some pages sharing the same entry point). +This entry point contains the page-level logic needed for that page and imports other modules for shared logic. -This manifests as an error during static files handling, referencing -optimization, minification, or parsing. -Sometimes this is due to strictness in the requirejs parsing. -Most often this is a trailing comma in a list of function parameters. +A typical new module structure will look something like: -Errors also pop up due to certain syntax, including -`spread syntax `__ and -`optional chaining `__. -This is the result of requirejs depending on a version of uglify that depends on an old version of -esprima. See `here `__. -In third party libraries that are already minified, we can work around this by using ``empty:`` to -skip optimization (docs). This is done for Sentry `here `__. -For our own code, we have a `babel plugin for requirejs `__. -See `here `__. +:: -How do I know whether or not I’m working with RequireJS? --------------------------------------------------------- + import "commcarehq"; // REQUIRED at the top of every "entry point" + // This loads site-wide dependencies needed to run global navigation, modals, notifications, etc. -You are likely working with RequireJS, as most of HQ has been migrated. -However, several major areas have **not** been migrated: app manager, -reports, and web apps. Test code also does not currently use RequireJS; -see -`Testing `__ -for working with tests. + // Common third-party dependencies + import $ from "jquery"; // Ideally, new pages should move away from jQuery and use native + // javascript. But this is here as an example. + import ko from "knockout"; + import "hqwebapp/js/knockout_bindings.ko"; // This one doesn't need a named parameter because it only adds + // knockout bindings and is not referenced in this file + import _ from "underscore"; + + // A commonly used internal module for passing server-side data to the front end + import initialPageData from "hqwebapp/js/initial_page_data"; + + /* page-level entry point logic begins here */ + + + +To register your module as a Webpack entry point, add the ``webpack_main`` template tag to your HTML template, +near the top and outside of any other block: -To tell for sure, look at your module’s ``hqDefine`` call, at the top of -the file. +:: + + {% webpack_main 'prototype/js/example' %} + +Some pages don't have any unique logic but do rely on other modules. +These are usually pages that use some common widgets but don't have custom UI interactions. + +If your page only relies on a single JavaScript module, you can use that as that +page's entry point: + +:: + + {% webpack_main 'locations/js/widgets' %} + +If your page relies on multiple modules, it still needs one entry point. +You can handle this by making a module that only imports other modules. +For instance an entry point located at ``prototype/js/combined_example.js`` +might look like: + +:: + + import "commcarehq"; // always at the top + + import "hqwebapp/js/crud_paginated_list_init"; + import "hqwebapp/js/bootstrap5/widgets"; + + // No page-specific logic, just need to collect the dependencies above + +Then in your HTML page: + +:: -RequireJS modules look like this, with all dependencies loaded as part -of ``hqDefine``: + {% webpack_main 'prototype/js/combined_example' %} + +The exception to the above is if your page inherits from a legacy page that +doesn't use a JavaScript bundler. This is rare, but one example would be adding a +new page to app manager that inherits from ``managed_app.html``. + + +Why is old code formatted differently? +-------------------------------------- + +You may notice that older JavaScript code on HQ is written in a modified AMD +(Asynchronous Module Definition) format. AMD was the only module format compatible +with our previous bundler, RequireJS. Most older entry points are written in this +modified AMD style and should eventually be migrated to an ESM format +once these entry points `are migrated to Webpack +`__. + +However, be careful when migrating modified AMD modules that aren't entry points, as some of these modules, +like ``hqwebapp/js/initial_page_data``, are still being referenced by pages not using a JavaScript bundler. +These pages still require this modified AMD approach until they transition to using a bundler. + +We will cover what common modified AMD modules look like in this section, but you can read more +about this choice of module format in the `Historical Background on Module Patterns +`__ + +The process of migrating a module from AMD to ESM is very straightforward. To learn more, +please see `Migrating Modules from AMD to ESM +`__ + + +Modified AMD Legacy Modules +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Modified AMD-style modules are used both on bundler pages and no-bundler pages. + +To differentiate between the two, look at the module's ``hqDefine`` call at the top of the file. + +Modules in this format used with Webpack (or RequireJS) will look like the following, +with all dependencies loaded as part of ``hqDefine``: :: @@ -81,10 +154,10 @@ of ``hqDefine``: ... }); -Non-RequireJS modules look like this, with no list and no function -parameters. HQ modules are loaded using ``hqImport`` in the body, and -third party libraries aren’t declared at all, instead relying on -globals: +In no-bundler areas of the codebase, "transition" AMD modules look like the following, +having no dependency list and no function parameters. +Additionally, HQ modules are loaded using ``hqImport`` in the body, and third-party libraries aren't declared at all, +instead relying on globals like ``ko`` (for Knockout.js) in the example below. :: @@ -93,102 +166,80 @@ globals: ... }); -How do I write a new page? --------------------------- -New code should be written in RequireJS, which is oriented around a -single “entry point” into the page. +How do I know whether or not I’m working with Webpack or RequireJS? +------------------------------------------------------------------- -Most pages have some amount of logic only relevant to that page, so they -have a file that includes that logic and then depends on other modules -for shared logic. +You are likely working with either Webpack or RequireJS, as most of HQ has been migrated to use a bundler. +However, several major areas have **not** been migrated: app manager, +reports, and web apps. -`data_dictionary.js `__ -fits this common pattern: +The easiest way to determine if a page is using either Webpack or RequireJS is to +open the JavaScript console on that page and type ``window.USE_WEBPACK``, which will return +``true`` if the page is using Webpack, or ``window.USE_REQUIREJS``, which will return +``true`` if the page is using RequireJS. If neither are ``true``, then the page is +a no-bundler page. -:: +ES Modules (ESM) +~~~~~~~~~~~~~~~~ - hqDefine("data_dictionary/js/data_dictionary", [ // Module name must match filename - "jquery", // Common third-party dependencies - "knockout", - "underscore", - "hqwebapp/js/initial_page_data", // Dependencies on HQ files always match the file's path - "hqwebapp/js/main", - "analytix/js/google", - "hqwebapp/js/knockout_bindings.ko", // This one doesn't need a named parameter because it only adds - // knockout bindings and is not referenced in this file - ], function ( - $, // These common dependencies use these names for compatibility - ko, // with non-requirejs pages, which rely on globals - _, - initialPageData, // Any dependency that will be referenced in this file needs a name. - hqMain, - googleAnalytics - ) { - /* Function definitions, knockout model definitions, etc. */ +If your page is using ESM, it is using Webpack, as RequireJS and no-bundler pages do +not use this module format. - var dataUrl = initialPageData.reverse('data_dictionary_json'); // Refer to dependencies by their named parameter - ... +ESM can quickly be identified by scanning the file for ``import`` statements like this: - $(function () { - /* Logic to run on documentready */ - }); +:: - // Other code isn't going to depend on this module, so it doesn't return anything or returns 1 - }); + import myDependency from "hqwebapp/js/my_dependency"; -To register your module as the RequireJS entry point, add the -``requirejs_main`` template tag to your HTML page, near the top but -outside of any other block: + import { Modal } from "bootstrap5"; -:: - {% requirejs_main 'data_dictionary/js/data_dictionary' %} +How do I add a new internal module or external dependency to an existing page? +------------------------------------------------------------------------------ -Some pages don’t have any unique logic but do rely on other modules. -These are usually pages that use some common widgets but don’t have -custom UI interactions. +Webpack supports multiple module formats, with ES Modules (ESM) being the preferred format. New modules should be written in the ESM format. -If your page only relies on a single js module, you can use that as the -module’s entry point: +That being said, a lot of legacy code on HQ is written in a modified AMD format. +If you are adding a lot of new code to such a module, it is recommended that you +`migrate this module to ESM format +<`__. +However, not every modified AMD module is ready to be migrated to ESM immediately, +so it's worth familiarizing yourself with working in that format. -:: +The format of the module you add a dependency to will determine how you include that dependency. - {% requirejs_main 'locations/js/widgets' %} +ESM Module +~~~~~~~~~~ -If your page relies on multiple modules, it still needs one entry point. -You can handle this by making a module that has no body, just a set of -dependencies, like in -`gateway_list.js `__: +ESM modules provide an extensive and flexible away of managing and naming imports from dependencies. :: - hqDefine("sms/js/gateway_list", [ - "hqwebapp/js/crud_paginated_list_init", - "hqwebapp/js/bootstrap3/widgets", - ], function () { - // No page-specific logic, just need to collect the dependencies above - }); + import myDependency from "hqwebapp/js/my_new_dependency"; + myDependency.myFunction(); -Then in your HTML page: + // using only portions of an dependency + import { Modal } from "bootstrap5"; + const myModal = new Modal(document.getElementById('#myModal')); -:: + // this also works + import bootstrap from "bootstrap5"; + const myOtherModal = new bootstrap.Modal(document.getElementById('#myOtherModal')); - {% requirejs_main 'sms/js/gateway_list' %} + // you can also alias imports + import * as myAliasedDep from "hqwebapp/js/my_other_dependency"; -The exception to the above is if your page inherits from a page that -doesn’t use RequireJS. This is rare, but one example would be adding a -new page to app manager that inherits from ``managed_app.html``. -How do I add a new dependency to an existing page? --------------------------------------------------- +Modified AMD (previously "RequireJS") +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -RequireJS -~~~~~~~~~ +.. warning:: + RequireJS is being replaced by Webpack. You should NOT create NEW modules with this style. -Add the new module to your module’s ``hqDefine`` list of dependencies. -If the new dependency will be directly referenced in the body of the -module, also add a parameter to the ``hqDefine`` callback: +To use your new module/dependency, add it your module’s ``hqDefine`` list of dependencies. +If the new dependency will be directly referenced in the body of the odule, also add a +parameter to the ``hqDefine`` callback: :: @@ -203,8 +254,21 @@ module, also add a parameter to the ``hqDefine`` callback: myDependency.myFunction(); }); -Non-RequireJS -~~~~~~~~~~~~~ + +No-Bundler Pages +~~~~~~~~~~~~~~~~ + +.. note:: + + No-Bundler pages are pages that do not have a Webpack (or RequireJS) entry point. + New pages should never be created without a ``webpack_main`` entry point. + + Eventually, the remaining pages in this category will be modularized properly to integrate with Webpack + as part of the `JS Bundler Migration + `__. + + Also note that these pages are **only** compatible with legacy modified AMD modules. ESM modules + do not work here. In your HTML template, add a script tag to your new dependency. Your template likely already has scripts included in a ``js`` block: @@ -227,13 +291,94 @@ dependency: myDependency.myFunction(); }); -Do **not** add the RequireJS-style dependency list and parameters. It’s +Do **not** add the dependency list and parameters from the modified AMD style or +use `hqImport` on ESM formatted modules. It's easy to introduce bugs that won’t be visible until the module is actually migrated, and migrations are harder when they have pre-existing -bugs. See the `troubleshooting section of the RequireJS Migration +bugs. See the `troubleshooting section of the JS Bundler Migration Guide `__ if you’re curious about the kinds of issues that crop up. + +My python tests are failing because of javascript +------------------------------------------------- + +Failures after "Building Webpack" +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The JavaScript tests run in Github Actions ``yarn build`` to check that ``webpack`` is building +without errors. You can run ``yarn build`` locally to simulate any errors encountered by these tests. + +Since you are likely developing using ``yarn dev``, you should have already encountered the +build errors during development. However, if the development build of Webpack is running +without failures, please check the ``webpack/webpack.prod.js`` configuration for possible +issues if the error messages don't yield anything useful. + + +RequireJS Test Failures +~~~~~~~~~~~~~~~~~~~~~~~ + +`TestRequireJS +`__ +reads all of our javascript files, checking for common errors. + +These tests are naive. They don't parse JavaScript, they just run regexes based on expected coding patterns. +They use `this method <#amd-style-legacy-modules>`__ to determine if a file is using an AMD module compatible +with a bundler (originally, RequireJS). This is one reason not to add dependency lists in areas of HQ +that don't yet use a bundler. + +**test_requirejs_disallows_hqimport** + +``hqImport`` only works in non-bundled contexts. In modules (Webpack or RequireJS), dependencies should be +included using ESM ``imports`` or listed module's ``hqDefine`` call, as described `here <#amd-style-legacy-modules>`__. + +Occasionally, this does not work due to a circular dependency. This will manifest as the module being undefined. +`hqRequire `__ +exists for this purpose, to require the necessary module at the point where it’s used. ``hqRequire`` defines +a new module, which can be fragile, so limit the code using it. As in python, best practice is to include +dependencies at the module level, at the top of the file. + + +**test_files_match_modules** + +RequireJS requires that a module's name is the same as the file containing it. Rename your module. + + +My deploy is failing because of javascript +------------------------------------------ + +Webpack Failures +~~~~~~~~~~~~~~~~ + +Webpack failures during deploy should be rare if you were able to run ``yarn dev`` successfully +locally during development. However, if these failures do occur, it is likely due to +issues with supporting deployment infrastructure. + +Is the version of ``npm`` and ``yarn`` up-to-date on the deploy machines? Are the supporting scripts +outlined in the staticfiles_collect tasks for `Webpack +`__ +configured properly? + + +RequireJS Failures +~~~~~~~~~~~~~~~~~~ + +This manifests as an error during static files handling, referencing +optimization, minification, or parsing. +Sometimes this is due to strictness in the requirejs parsing. +Most often this is a trailing comma in a list of function parameters. + +Errors also pop up due to certain syntax, including +`spread syntax `__ and +`optional chaining `__. +This is the result of requirejs depending on a version of uglify that depends on an old version of +esprima. See `here `__. +In third party libraries that are already minified, we can work around this by using ``empty:`` to +skip optimization (docs). This is done for Sentry `here `__. +For our own code, we have a `babel plugin for requirejs `__. +See `here `__. + + How close are we to a world where we’ll just have one set of conventions? ------------------------------------------------------------------------- @@ -242,24 +387,14 @@ significant complexity. `hqDefine.sh `__ generates metrics for the current status of the migration and locates -umigrated files. At the time of writing: +un-migrated files. At the time of writing: :: - $ ./scripts/codechecks/hqDefine.sh - - 97% (1040/1081) of HTML files are free of inline scripts - 93% (501/539) of JS files use hqDefine - 64% (342/539) of JS files specify their dependencies - 93% (995/1080) of HTML files are free of script tags - -Why aren’t we using something more fully-featured, more modern, or cooler than RequireJS? ------------------------------------------------------------------------------------------ - -RequireJS is now `deprecated `__. + $ ./scripts/codechecks/hqDefine.sh -This migration began quite a while ago. At the time, the team discussed -options and selected RequireJS. The majority of the work done to move to -RequireJS has been around reorganizing code into modules and explicitly -declaring dependencies, which is necessary for any kind of modern -dependency management. + 97% (1209/1254) of HTML files are free of inline scripts + 94% (553/594) of non-ESM JS files use hqDefine + 75% (443/594) of non-ESM JS files specify their dependencies + 93% (1162/1254) of HTML files are free of script tags + 1% (3/597) of JS files use ESM format diff --git a/docs/js-guide/migrating.rst b/docs/js-guide/migrating.rst index 945df0b5691c..45a71718c79c 100644 --- a/docs/js-guide/migrating.rst +++ b/docs/js-guide/migrating.rst @@ -1,8 +1,11 @@ +JS Bundler Migration Guide +=========================== -RequireJS Migration Guide -========================= +This page is a guide to upgrading legacy code in HQ to use Webpack, a modern JavaScript bundler. +Previously, we were migrating legacy code to RequireJS, an older JavaScript bundler which has since been +deprecated. If you wish to migrate a RequireJS page to Webpack, please see the `RequireJS to Webpack Migration Guide +`__ -This page is a guide to upgrading legacy code in HQ to use RequireJS. For information on how to work within existing code, see `Managing Dependencies `__. Both that page and `Historical Background on Module @@ -16,20 +19,22 @@ are useful background for this guide. Background: modules and pages ----------------------------- -The RequireJS migration deals with both **pages** (HTML) and **modules** +The JS Bundler migration deals with both **pages** (HTML) and **modules** (JavaScript). Any individual page is either migrated or not. Individual modules are also migrated or not, but a “migrated” module may be used on -both RequireJS and non-RequireJS pages. - -Logic in ``hqModules.js`` determines whether or not we’re in a RequireJS -environment and changes the behavior of ``hqDefine`` accordingly. In a -RequireJS environment, ``hqDefine`` just passes through to RequireJS’s -``define``. Once all pages have been migrated, we’ll be able to delete +both bundled (Webpack and RequireJS) and non-bundled pages. + +Logic in ``hqModules.js`` determines whether or not we’re in a bundler +environment (Webpack or RequireJS) and changes the behavior of +``hqDefine`` accordingly. In a bundler environment, ``hqDefine`` just passes +through to the standard AMD Module ``define``, which is understood by +both Webpack and RequireJS as a module and bundled accordingly. +Once all pages have been migrated, we’ll be able to delete ``hqModules.js`` altogether and switch all of the ``hqDefine`` calls to -``define``. +``define``--or better, switch to using ES Modules (ESM). These docs walk through the process of migrating a single page to -RequireJS. +from not using a JavaScript bundler to using Webpack. Basic Migration Process ----------------------- @@ -41,66 +46,77 @@ in HQ. Pages that are not descendants of `hqwebapp/base.html `__, which are rare, cannot yet be migrated. -Once these conditions are met, migrating to RequireJS is essentially the +Once these conditions are met, migrating to Webpack is essentially the process of explicitly adding each module’s dependencies to the module’s definition, and also updating each HTML page to reference a single “main” module rather than including a bunch of ``{% endif %}`` + ``{% if webpack_main %}{% endif %}`` - Also check the parent’s parent template, etc. Stop once you get to - ``hqwebapp/base.html``, ``hqwebapp/bootstrap3/two_column.html``, or - ``hqwebapp/bootstrap3/base_section.html``, which already support requirejs. + ``hqwebapp/base.html``, ``hqwebapp/bootstrap5/two_column.html``, or + ``hqwebapp/bootstrap5/base_section.html``, which already support a bundler. - Check the view for any `hqwebapp decorators `__ like ``use_jquery_ui`` which are used to include many common yet not @@ -158,7 +176,7 @@ To declare dependencies: a dependency and also add the global to ``thirdPartyGlobals`` in `hqModules.js `__ which prevents errors on pages that use your module but are not yet - migrated to requirejs. + migrated to Webpack. Dependencies that aren’t directly referenced as modules **don’t** need to be added as function parameters, but they **do** need to be in the @@ -178,9 +196,9 @@ based on the changes you made: - If you replaced any ``hqImport`` calls that were inside of event handlers or other callbacks, verify that those areas still work correctly. When a migrated module is used on an - unmigrated page, its dependencies need to be available at the time the + un-migrated page, its dependencies need to be available at the time the module is defined. This is a change from previous behavior, where the - dependencies didn’t need to be defined until ``hqImport`` first called + dependencies didn't need to be defined until ``hqImport`` first called them. We do not currently have a construct to require dependencies after a module is defined. - The most likely missing dependencies are the @@ -188,8 +206,9 @@ based on the changes you made: often don’t error but will look substantially different on the page if they haven’t been initialized. - If your page depends on any third-party - modules that might not yet be used on any RequireJS pages, test them. - Third-party modules sometimes need to be upgraded to be compatible with RequireJS. + modules that might not yet be used on any Webpack pages, test them. + Third-party modules sometimes need to be upgraded to be compatible with Webpack + or "shimmed" using Webpack's ``exports-loader``. - If your page touched any javascript modules that are used by pages that haven’t yet been migrated, test at least one of those non-migrated pages. @@ -201,11 +220,13 @@ Troubleshooting Troubleshooting migration issues ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -When debugging RequireJS issues, the first question is whether or not +When debugging Webpack issues, the first question is whether or not the page you’re on has been migrated. You can find out by checking the -value of ``window.USE_REQUIREJS`` in the browser console. +value of ``window.USE_WEBPACK`` in the browser console or ``window.USE_REQUIREJS`` +if the page is still using RequireJS. If neither values return ``true``, then +the page has not been migrated yet. -Common issues on RequireJS pages: +Common issues on Webpack pages: - JS error like ``$(...).something is not a function``: this indicates there’s a missing @@ -215,34 +236,30 @@ Common issues on RequireJS pages: - Missing functionality, but no error: this usually indicates a missing knockout binding. To fix, add the file containing the binding to the module that applies that binding, which - usually means adding ``hqwebapp/js/knockout_bindings.ko`` to the page’s main module. + usually means adding ``hqwebapp/js/knockout_bindings.ko`` to the page’s entry point. - JS error like ``something is not defined`` where ``something`` is one of the parameters in the module’s main function: this can indicate a circular dependency. This is rare in HQ. Track down the circular dependency and see if it makes sense to eliminate it by - reorganizing code. If it doesn’t, you can use + reorganizing code. If it doesn’t work, you can use `hqRequire `__ to require the necessary module at the point where it’s used rather than at the top of the module using it. - JS error like ``x is not defined`` where ``x`` is a third-party module, which is the dependency of another - third party module ``y`` and both of them are non RequireJs modules. You + third party module ``y`` and both of them are not modules. You may get this intermittent error when you want to use ``y`` in the - migrated module and ``x`` and ``y`` does not support - `AMD `__. You can fix this using - `shim `__ - or - `hqRequire `__. - `Example `__ - of this could be ``d3`` and ``nvd3`` + migrated module and ``x`` and ``y`` is not formatted as a recognizable JavaScript module that + Webpack recognizes (AMD, ESM, CommonJS are supported formats). You can fix this by adding + an ``exports-loader`` ``rule`` `like this example `__. -Common issues on non-RequireJS pages: +Common issues on non-Bundled pages: - JS error like ``something is not defined`` where ``something`` is a third-party - module: this can happen if a non-RequireJS page uses a RequireJS module + module: this can happen if a non-Bundled page uses a Bundled module which uses a third party module based on a global variable. There’s some - code that mimicks RequireJS in this situation, but it needs to know + code that mimics AMD modules in this situation, but it needs to know about all of the third party libraries. To fix, add the third party module’s global to `thirdPartyMap in hqModules.js `__. @@ -251,7 +268,7 @@ Common issues on non-RequireJS pages: appears before one of its dependencies. This can happen to migrated modules because one of the effects of the migration is to typically import all of a module’s dependencies at the time the module is defined, - which in a non-RequireJS context means all of the dependencies’ script + which in a non-bundled context means all of the dependencies’ script tags must appear before the script tags that depend on them. Previously, dependencies were not imported until ``hqImport`` was called, which could be later on, possibly in an event handler or some other code that @@ -262,6 +279,10 @@ Common issues on non-RequireJS pages: Troubleshooting the RequireJS build process ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. note:: + This is here for informational purposes only and will be removed once + all of the RequireJS code is moved to Webpack. + Tactics that can help track down problems with the RequireJS build process, which usually manifest as errors that happen on staging but not locally: diff --git a/docs/js-guide/module-history.rst b/docs/js-guide/module-history.rst index b7409b6dc699..6f6f04da3eda 100644 --- a/docs/js-guide/module-history.rst +++ b/docs/js-guide/module-history.rst @@ -7,30 +7,44 @@ Dependencies `__. Part of this process has -included developing a lighter-weight alternative module system called -``hqDefine``. +We're in the process of migrating to +`Webpack `__. Prior to the migration to Webpack, +we were migrating No-Bundler pages to RequireJS, which has since become deprecated. +Part of this ongoing process has included developing a lighter-weight +alternative module system called ``hqDefine``, that is based on the AMD (Asynchronous +Module Definition) format. This ``hqDefine`` variant of AMD is relevant even for +Webpack bundles, as it serves as the "transition" module format between pages using +a bundler and pages not using a bundler (Legacy Pages). ``hqDefine`` serves as a stepping stone between legacy code and -requirejs modules: it adds encapsulation but not full-blown dependency -management. New code is written in RequireJS, but ``hqDefine`` exists to -support legacy code that does not yet use RequireJS. +bundled code: it adds encapsulation but not full-blown dependency +management. New code is written in Webpack, but ``hqDefine`` exists to +support legacy code that does not yet use Webpack. Before diving into ``hqDefine``, I want to talk first about the status -quo convention for sanity with no module system. As we’ll describe, it’s +quo convention for sanity with no module system. As we'll describe, it's a step down from our current preferred choice, but it’s still miles ahead of having no convention at all. +Creating a Brand New Page? +-------------------------- + +If you are creating new pages with Webpack, then you can skip this +discussion for now. All **new** Webpack entry points should be written using +the new ES Module (ESM) format. This is the modern module format you will +see referenced in most modern JavaScript library documentation. ``hqDefine`` +(aka Modified AMD) should eventually only be used for transition points +between legacy code and bundled code. + The Crockford Pattern --------------------- @@ -127,7 +141,7 @@ hqDefine There are many great module systems out there, so why did we write our own? The answer’s pretty simple: while it’s great to start with -require.js or system.js, with a code base HQ’s size, getting from here +Webpack, with a code base HQ’s size, getting from here to there is nearly impossible without an intermediate step. Using the above example again, using ``hqDefine``, you’d write your file @@ -164,8 +178,9 @@ function itself is exactly the same. It’s just being passed to ``hqDefine`` instead of being called directly. ``hqDefine`` is an intermediate step on the way to full support for AMD -modules, which in HQ is implemented using RequireJS. ``hqDefine`` checks -whether or not it is on a page that uses AMD modules and then behaves in +modules, which is supported by Webpack as well as our previous bundler RequireJS. + +``hqDefine`` checks whether or not it is on a page that uses AMD modules and then behaves in one of two ways: \* If the page has been migrated, meaning it uses AMD modules, ``hqDefine`` just delegates to ``define``. \* If the page has not been migrated, ``hqDefine`` acts as a thin wrapper around the @@ -173,7 +188,7 @@ Crockford module pattern. ``hqDefine`` takes a function, calls it immediately, and puts it in a namespaced global; ``hqImport`` then looks up the module in that global. -In the first case, by handing control over to RequireJS, +In the first case, by handing control over to Webpack, ``hqDefine``/``hqImport`` also act as a module *loader*. But in the second case, they work only as a module *dereferencer*, so in order to use a module, it still needs to be included as a ``