diff --git a/CHANGELOG.md b/CHANGELOG.md index 49d48c1329..da771daf8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,13 @@ All notable changes to this project will be documented in this file. The format This project does NOT adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and major versions of this project denote compatibility with Sitecore Platform versions. Refer to the "Headless Services" section in the [Sitecore modules compatibility table](https://support.sitecore.com/kb?id=kb_article_view&sysparm_article=KB1000576) or the [Headless Rendering download page](https://dev.sitecore.net/Downloads/Sitecore_Headless_Rendering.aspx) for more details on versioning. +## 20.2.0 + +### 🎉 New Features & Improvements + +* `[sitecore-jss]` `[templates/nextjs]` GraphQL Layout and Dictionary services can handle endpoint rate limits through retryer functionality in GraphQLClient. To prevent SSG builds from failing and enable multiple retries, set retry amount in lib/dictionary-service-factory and lib/layout-service-factory ([commit](https://github.com/Sitecore/jss/pull/1631/commits/d39d74ad7bbeddcb66b7de4377070e178851abc5))([#1631](https://github.com/Sitecore/jss/pull/1631)) +* `[sitecore-jss-nextjs]` Reduce the amount of Edge API calls during fetch getStaticPaths ([commit](https://github.com/Sitecore/jss/pull/1631/commits/cd2771b256ac7c38818ee6bea48278958ac455ca))([#1631](https://github.com/Sitecore/jss/pull/1631)) + ## 20.1.0 ### 🎉 New Features & Improvements diff --git a/packages/create-sitecore-jss/src/templates/nextjs-sxa/src/lib/page-props-factory/plugins/error-pages.ts b/packages/create-sitecore-jss/src/templates/nextjs-sxa/src/lib/page-props-factory/plugins/error-pages.ts index ffe06eb737..af84245991 100644 --- a/packages/create-sitecore-jss/src/templates/nextjs-sxa/src/lib/page-props-factory/plugins/error-pages.ts +++ b/packages/create-sitecore-jss/src/templates/nextjs-sxa/src/lib/page-props-factory/plugins/error-pages.ts @@ -14,6 +14,10 @@ class ErrorPagesPlugin implements Plugin { apiKey: config.sitecoreApiKey, siteName: config.jssAppName, language: props.locale, + retries: + (process.env.GRAPH_QL_SERVICE_RETRIES && + parseInt(process.env.GRAPH_QL_SERVICE_RETRIES, 10)) || + 0, }); if (props.notFound) { diff --git a/packages/create-sitecore-jss/src/templates/nextjs/.env b/packages/create-sitecore-jss/src/templates/nextjs/.env index ccc7ecd3da..81bd93b76b 100644 --- a/packages/create-sitecore-jss/src/templates/nextjs/.env +++ b/packages/create-sitecore-jss/src/templates/nextjs/.env @@ -36,6 +36,9 @@ SITECORE_API_HOST= # the resolved Sitecore API hostname + the `graphQLEndpointPath` defined in your `package.json`. GRAPH_QL_ENDPOINT= +# How many times should GraphQL Layout, Dictionary and ErrorPages services retry a fetch when endpoint rate limit is reached +GRAPH_QL_SERVICE_RETRIES=0 + # The way in which layout and dictionary data is fetched from Sitecore FETCH_WITH=<%- fetchWith %> diff --git a/packages/create-sitecore-jss/src/templates/nextjs/src/lib/dictionary-service-factory.ts b/packages/create-sitecore-jss/src/templates/nextjs/src/lib/dictionary-service-factory.ts index 25616ccafb..e53cee33cc 100644 --- a/packages/create-sitecore-jss/src/templates/nextjs/src/lib/dictionary-service-factory.ts +++ b/packages/create-sitecore-jss/src/templates/nextjs/src/lib/dictionary-service-factory.ts @@ -18,6 +18,17 @@ export class DictionaryServiceFactory { otherwise, the service will attempt to figure out the root item for the current JSS App using GraphQL and app name. rootItemId: '{GUID}' */ + /* + GraphQL endpoint may reach its rate limit with the amount of Layout and Dictionary requests it receives and throw a rate limit error. + GraphQL Dictionary and Layout Services can handle rate limit errors from server and attempt a retry on requests. + For this, specify the number of retries the GraphQL client will attempt. + It will only try the request once by default. + retries: 'number' + */ + retries: + (process.env.GRAPH_QL_SERVICE_RETRIES && + parseInt(process.env.GRAPH_QL_SERVICE_RETRIES, 10)) || + 0, }) : new RestDictionaryService({ apiHost: config.sitecoreApiHost, diff --git a/packages/create-sitecore-jss/src/templates/nextjs/src/lib/layout-service-factory.ts b/packages/create-sitecore-jss/src/templates/nextjs/src/lib/layout-service-factory.ts index a3e466ae31..48812367ec 100644 --- a/packages/create-sitecore-jss/src/templates/nextjs/src/lib/layout-service-factory.ts +++ b/packages/create-sitecore-jss/src/templates/nextjs/src/lib/layout-service-factory.ts @@ -12,6 +12,17 @@ export class LayoutServiceFactory { endpoint: config.graphQLEndpoint, apiKey: config.sitecoreApiKey, siteName: config.jssAppName, + /* + GraphQL endpoint may reach its rate limit with the amount of Layout and Dictionary requests it receives and throw a rate limit error. + GraphQL Dictionary and Layout Services can handle rate limit errors from server and attempt a retry on requests. + For this, specify the number of retries the GraphQL client will attempt. + It will only try the request once by default. + retries: 'number' + */ + retries: + (process.env.GRAPH_QL_SERVICE_RETRIES && + parseInt(process.env.GRAPH_QL_SERVICE_RETRIES, 10)) || + 0, }) : new RestLayoutService({ apiHost: config.sitecoreApiHost, diff --git a/packages/sitecore-jss-nextjs/src/services/graphql-sitemap-service.test.ts b/packages/sitecore-jss-nextjs/src/services/graphql-sitemap-service.test.ts index 3abccb0142..1367b6b882 100644 --- a/packages/sitecore-jss-nextjs/src/services/graphql-sitemap-service.test.ts +++ b/packages/sitecore-jss-nextjs/src/services/graphql-sitemap-service.test.ts @@ -232,6 +232,17 @@ describe('GraphQLSitemapService', () => { }); }); + it('should throw error if empty language is provided', async () => { + mockPathsRequest(); + + const service = new GraphQLSitemapService({ endpoint, apiKey, siteName }); + await service.fetchSSGSitemap(['']).catch((error: RangeError) => { + expect(error.message).to.equal('The language must be a non-empty string'); + }); + + return expect(nock.isDone()).to.be.false; + }); + it('should use a custom pageSize, if provided', async () => { const customPageSize = 20; @@ -257,7 +268,7 @@ describe('GraphQLSitemapService', () => { .post( '/', (body) => - body.query.indexOf('$pageSize: Int = 10') > 0 && body.variables.pageSize === undefined + body.query.indexOf('$pageSize: Int = 100') > 0 && body.variables.pageSize === undefined ) .reply(200, sitemapQueryResult); diff --git a/packages/sitecore-jss-nextjs/src/services/graphql-sitemap-service.ts b/packages/sitecore-jss-nextjs/src/services/graphql-sitemap-service.ts index 5461f9af21..1182a36c9a 100644 --- a/packages/sitecore-jss-nextjs/src/services/graphql-sitemap-service.ts +++ b/packages/sitecore-jss-nextjs/src/services/graphql-sitemap-service.ts @@ -14,12 +14,15 @@ export const queryError = /** @private */ export const languageError = 'The list of languages cannot be empty'; +/** @private */ +const languageEmptyError = 'The language must be a non-empty string'; + // Even though _hasLayout should always be "true" in this query, using a variable is necessary for compatibility with Edge const defaultQuery = /* GraphQL */ ` query SitemapQuery( $rootItemId: String! $language: String! - $pageSize: Int = 10 + $pageSize: Int = 100 $hasLayout: String = "true" $after: String ) { @@ -173,23 +176,29 @@ export class GraphQLSitemapService { throw new Error(queryError); } - // Fetch paths using all locales - const paths = await Promise.all( - languages.map((language) => { - debug.sitemap('fetching sitemap data for %s', language); - return this.searchService - .fetch(this.query, { - rootItemId, - language, - pageSize: this.options.pageSize, - }) - .then((results) => { - return results.map((item) => - formatStaticPath(item.url.path.replace(/^\/|\/$/g, '').split('/'), language) - ); - }); - }) - ); + const paths: StaticPath[] = []; + + for (const language of languages) { + if (language === '') { + throw new RangeError(languageEmptyError); + } + + debug.sitemap('fetching sitemap data for %s', language); + + const languagePaths = await this.searchService + .fetch(this.query, { + rootItemId, + language, + pageSize: this.options.pageSize, + }) + .then((results) => { + return results.map((item) => + formatStaticPath(item.url.path.replace(/^\/|\/$/g, '').split('/'), language) + ); + }); + + paths.push(...languagePaths); + } // merge promises results into single result return ([] as StaticPath[]).concat(...paths); diff --git a/packages/sitecore-jss/src/graphql-request-client.test.ts b/packages/sitecore-jss/src/graphql-request-client.test.ts index 26e73abdef..92ebb935d4 100644 --- a/packages/sitecore-jss/src/graphql-request-client.test.ts +++ b/packages/sitecore-jss/src/graphql-request-client.test.ts @@ -1,3 +1,4 @@ +/* eslint-disable dot-notation */ /* eslint-disable no-unused-expressions */ import { expect, use, spy } from 'chai'; import spies from 'chai-spies'; @@ -117,4 +118,63 @@ describe('GraphQLRequestClient', () => { ); } }); + + it('should use retry and throw error when retries specified', async function() { + this.timeout(6000); + nock('http://jssnextweb') + .post('/graphql') + .reply(429) + .post('/graphql') + .reply(429) + .post('/graphql') + .reply(429); + const graphQLClient = new GraphQLRequestClient(endpoint, { retries: 2 }); + spy.on(graphQLClient['client'], 'request'); + await graphQLClient.request('test').catch((error) => { + expect(error).to.not.be.undefined; + expect(graphQLClient['client'].request).to.be.called.exactly(3); + spy.restore(graphQLClient); + }); + }); + + it('should use retry and resolve if one of the requests resolves', async function() { + this.timeout(6000); + nock('http://jssnextweb') + .post('/graphql') + .reply(429) + .post('/graphql') + .reply(429) + .post('/graphql') + .reply(200, { + data: { + result: 'Hello world...', + }, + }); + const graphQLClient = new GraphQLRequestClient(endpoint, { retries: 3 }); + spy.on(graphQLClient['client'], 'request'); + + const data = await graphQLClient.request('test'); + + expect(data).to.not.be.null; + expect(graphQLClient['client'].request).to.be.called.exactly(3); + spy.restore(graphQLClient); + }); + + it('should use [retry-after] header value when response is 429', async function() { + this.timeout(6000); + nock('http://jssnextweb') + .post('/graphql') + .reply(429, {}, { 'Retry-After': '2' }); + const graphQLClient = new GraphQLRequestClient(endpoint, { retries: 1 }); + spy.on(graphQLClient, 'debug'); + + await graphQLClient.request('test').catch(() => { + expect(graphQLClient['debug']).to.have.been.called.with( + 'Error: Rate limit reached for GraphQL endpoint. Retrying in %ds. Retries left: %d', + 2, + 1 + ); + spy.restore(graphQLClient); + }); + }); }); diff --git a/packages/sitecore-jss/src/graphql-request-client.ts b/packages/sitecore-jss/src/graphql-request-client.ts index f6ffb928e9..71d40b4c6c 100644 --- a/packages/sitecore-jss/src/graphql-request-client.ts +++ b/packages/sitecore-jss/src/graphql-request-client.ts @@ -31,6 +31,10 @@ export type GraphQLRequestClientConfig = { * Override fetch method. Uses 'graphql-request' library default otherwise ('cross-fetch'). */ fetch?: typeof fetch; + /** + * Number of retries for client. Will be used if endpoint responds with 429 (rate limit reached) error + */ + retries?: number; }; /** @@ -41,6 +45,7 @@ export class GraphQLRequestClient implements GraphQLClient { private client: Client; private headers: Record = {}; private debug: Debugger; + private retries: number; /** * Provides ability to execute graphql query using given `endpoint` @@ -58,6 +63,7 @@ export class GraphQLRequestClient implements GraphQLClient { ); } + this.retries = clientConfig.retries || 0; this.client = new Client(endpoint, { headers: this.headers, fetch: clientConfig.fetch }); this.debug = clientConfig.debugger || debuggers.http; } @@ -71,7 +77,9 @@ export class GraphQLRequestClient implements GraphQLClient { query: string | DocumentNode, variables?: { [key: string]: unknown } ): Promise { - return new Promise((resolve, reject) => { + let retriesLeft = this.retries; + + const retryer = async (): Promise => { // Note we don't have access to raw request/response with graphql-request // (or nice hooks like we have with Axios), but we should log whatever we have. this.debug('request: %o', { @@ -81,16 +89,36 @@ export class GraphQLRequestClient implements GraphQLClient { variables, }); - this.client + return this.client .request(query, variables) .then((data: T) => { this.debug('response: %o', data); - resolve(data); + return data; }) .catch((error: ClientError) => { this.debug('response error: %o', error.response); - return reject(error); + + if (error.response?.status === 429 && retriesLeft > 0) { + const rawHeaders = (error as ClientError)?.response?.headers; + const delaySeconds = + rawHeaders && rawHeaders.get('Retry-After') + ? Number.parseInt(rawHeaders.get('Retry-After'), 10) + : 1; + + this.debug( + 'Error: Rate limit reached for GraphQL endpoint. Retrying in %ds. Retries left: %d', + delaySeconds, + retriesLeft + ); + + retriesLeft--; + return new Promise((resolve) => setTimeout(resolve, delaySeconds * 1000)).then(retryer); + } + + throw error; }); - }); + }; + + return retryer(); } } diff --git a/packages/sitecore-jss/src/i18n/graphql-dictionary-service.ts b/packages/sitecore-jss/src/i18n/graphql-dictionary-service.ts index fd3dab86f4..b0a5d7f8b2 100644 --- a/packages/sitecore-jss/src/i18n/graphql-dictionary-service.ts +++ b/packages/sitecore-jss/src/i18n/graphql-dictionary-service.ts @@ -1,4 +1,8 @@ -import { GraphQLClient, GraphQLRequestClient } from '../graphql-request-client'; +import { + GraphQLClient, + GraphQLRequestClient, + GraphQLRequestClientConfig, +} from '../graphql-request-client'; import { SitecoreTemplateId } from '../constants'; import { DictionaryPhrases, DictionaryServiceBase } from './dictionary-service'; import { CacheOptions } from '../cache-client'; @@ -48,7 +52,10 @@ const query = /* GraphQL */ ` /** * Configuration options for @see GraphQLDictionaryService instances */ -export interface GraphQLDictionaryServiceConfig extends SearchServiceConfig, CacheOptions { +export interface GraphQLDictionaryServiceConfig + extends SearchServiceConfig, + CacheOptions, + Pick { /** * The URL of the graphQL endpoint. */ @@ -155,6 +162,7 @@ export class GraphQLDictionaryService extends DictionaryServiceBase { return new GraphQLRequestClient(this.options.endpoint, { apiKey: this.options.apiKey, debugger: debug.dictionary, + retries: this.options.retries, }); } } diff --git a/packages/sitecore-jss/src/layout/graphql-layout-service.ts b/packages/sitecore-jss/src/layout/graphql-layout-service.ts index 4adaeac927..e487582922 100644 --- a/packages/sitecore-jss/src/layout/graphql-layout-service.ts +++ b/packages/sitecore-jss/src/layout/graphql-layout-service.ts @@ -1,9 +1,13 @@ import { LayoutServiceBase } from './layout-service'; import { LayoutServiceData } from './models'; -import { GraphQLClient, GraphQLRequestClient } from '../graphql-request-client'; +import { + GraphQLClient, + GraphQLRequestClient, + GraphQLRequestClientConfig, +} from '../graphql-request-client'; import debug from '../debug'; -export type GraphQLLayoutServiceConfig = { +export type GraphQLLayoutServiceConfig = Pick & { /** * Your Graphql endpoint */ @@ -79,6 +83,7 @@ export class GraphQLLayoutService extends LayoutServiceBase { return new GraphQLRequestClient(this.serviceConfig.endpoint, { apiKey: this.serviceConfig.apiKey, debugger: debug.layout, + retries: this.serviceConfig.retries, }); } diff --git a/packages/sitecore-jss/src/site/graphql-error-pages-service.ts b/packages/sitecore-jss/src/site/graphql-error-pages-service.ts index 9d12d363e7..3da6106eea 100644 --- a/packages/sitecore-jss/src/site/graphql-error-pages-service.ts +++ b/packages/sitecore-jss/src/site/graphql-error-pages-service.ts @@ -1,4 +1,4 @@ -import { GraphQLClient, GraphQLRequestClient } from '../graphql'; +import { GraphQLClient, GraphQLRequestClient, GraphQLRequestClientConfig } from '../graphql'; import { siteNameError } from '../constants'; import debug from '../debug'; @@ -16,7 +16,7 @@ const defaultQuery = /* GraphQL */ ` } `; -export type GraphQLErrorPagesServiceConfig = { +export type GraphQLErrorPagesServiceConfig = Pick & { /** * Your Graphql endpoint */ @@ -98,6 +98,7 @@ export class GraphQLErrorPagesService { return new GraphQLRequestClient(this.options.endpoint, { apiKey: this.options.apiKey, debugger: debug.errorpages, + retries: this.options.retries, }); } }