Skip to content

Commit

Permalink
AG-31356 return rawFilter as a string
Browse files Browse the repository at this point in the history 
Merge in EXTENSIONS/filters-downloader from fix/AG-31356 to master

Squashed commit of the following:

commit b585c43
Author: Maxim Topciu <[email protected]>
Date:   Tue Mar 26 16:00:37 2024 +0200

    AG-31356 update changelog

commit 684cf07
Author: Maxim Topciu <[email protected]>
Date:   Mon Mar 25 16:01:17 2024 +0200

    AG-31356 add link

commit ae9b49d
Author: Maxim Topciu <[email protected]>
Date:   Mon Mar 25 16:00:14 2024 +0200

    AG-31356 fix changelog

commit bb34931
Merge: aaef844 2898d7c
Author: Maxim Topciu <[email protected]>
Date:   Mon Mar 25 15:55:11 2024 +0200

    Merge branch 'master' into fix/AG-31356

commit aaef844
Author: Maxim Topciu <[email protected]>
Date:   Mon Mar 25 13:36:26 2024 +0200

    AG-31356 return rawFilter as a string

commit d9861d6
Author: Slava Leleka <[email protected]>
Date:   Wed Mar 20 19:39:17 2024 +0200

    update changelog

commit d9ba34c
Author: Slava Leleka <[email protected]>
Date:   Wed Mar 20 19:38:55 2024 +0200

    update changelog

commit f0e75ed
Author: Slava Leleka <[email protected]>
Date:   Wed Mar 20 19:38:46 2024 +0200

    update readme

commit c6f50fa
Author: Slava Leleka <[email protected]>
Date:   Wed Mar 20 19:04:15 2024 +0200

    cleanup

commit fa78ee9
Author: Slava Leleka <[email protected]>
Date:   Tue Mar 19 17:44:13 2024 +0200

    handle a failed patch update
  • Loading branch information
@maximtop
maximtop committed Mar 26, 2024
1 parent 2898d7c commit a890013
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 61 deletions.
7 changes: 7 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [2.2.0] - 2024-03-26

### Changed

- rawFilter is returned as a string, so that new lines can be preserved.

[2.2.0]: https://github.com/AdguardTeam/FiltersDownloader/compare/v2.1.2...v2.2.0

## [2.1.2] - 2024-03-25

Expand Down
49 changes: 22 additions & 27 deletions __tests__/filters-downloader.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ describe('FiltersDownloader', () => {
const { filter, rawFilter } = await FiltersDownloader.downloadWithRaw(url, { rawFilter: prevFilter });
expect(filter).toEqual(curFilter.trim().split(/[\r\n]+/));
// since there are no any includes, they are the same
expect(rawFilter).toEqual(curFilter.trim().split(/[\r\n]+/));
expect(rawFilter).toEqual(curFilter);
});

it('applies conditions after patches', async () => {
Expand All @@ -55,13 +55,11 @@ describe('FiltersDownloader', () => {
'||example.net^',
'||included.com^',
];
const expectedRawFilter = [
'! Title: Diff Updates Simple Example List',
'! Version: v1.0.2',
'! Diff-Path: patches/v1.0.1-s-1702460925-5.patch',
'||example.net^',
'!#include ./filter_2.txt',
];
const expectedRawFilter = `! Title: Diff Updates Simple Example List
! Version: v1.0.2
! Diff-Path: patches/v1.0.1-s-1702460925-5.patch
||example.net^
!#include ./filter_2.txt`;
expect(filter).toEqual(expectedFilter);
expect(rawFilter).toEqual(expectedRawFilter);
});
Expand Down Expand Up @@ -102,13 +100,12 @@ describe('FiltersDownloader', () => {
'||example.net^',
'||included.com^',
];
const expectedRawFilter = [
'! Title: Diff Updates Simple Example List',
'! Version: v1.0.2',
'! Diff-Path: patches/v1.0.1-s-1702460925-5.patch',
'||example.net^',
'!#include filter_2.txt',
];
const expectedRawFilter = `! Title: Diff Updates Simple Example List
! Version: v1.0.2
! Diff-Path: patches/v1.0.1-s-1702460925-5.patch
||example.net^
!#include filter_2.txt
`;
expect(filter).toEqual(expectedFilter);
expect(rawFilter).toEqual(expectedRawFilter);
// Assert that each URL was fetched only once
Expand Down Expand Up @@ -161,12 +158,11 @@ describe('FiltersDownloader', () => {
'||example.net^',
'||included.com^',
];
const expectedRawFilter = [
'! Title: Simple Example List',
'! Version: v1.0.2',
'||example.net^',
'!#include filter_2.txt',
];
const expectedRawFilter = `! Title: Simple Example List
! Version: v1.0.2
||example.net^
!#include filter_2.txt
`;
expect(filter).toEqual(expectedFilter);
expect(rawFilter).toEqual(expectedRawFilter);
// Assert that each URL was fetched only once
Expand Down Expand Up @@ -221,12 +217,11 @@ describe('FiltersDownloader', () => {
'! Version: v1.0.0',
'||included2.com^',
];
const expectedRawFilter = [
'! Title: Simple Example List',
'! Version: v1.0.2',
'||example.net^',
'!#include filter_2.txt',
];
const expectedRawFilter = `! Title: Simple Example List
! Version: v1.0.2
||example.net^
!#include filter_2.txt
`;
expect(filter).toEqual(expectedFilter);
expect(rawFilter).toEqual(expectedRawFilter);
// Assert that each URL was fetched only once
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@adguard/filters-downloader",
"version": "2.1.2",
"version": "2.2.0",
"description": "Compiles filters source files",
"main": "dist/index.js",
"browser": "dist/index.browser.js",
Expand Down
55 changes: 22 additions & 33 deletions src/filters-downloader-creator.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ export interface DownloadResult {
/**
* The raw filter, which is the filter before any conditions are applied or inclusions resolved.
*/
rawFilter: string[];
rawFilter: string;

/**
* Flag to indicate if the patch update failed.
Expand Down Expand Up @@ -598,15 +598,12 @@ const FiltersDownloaderCreator = (FileDownloadWrapper: IFileDownloader): IFilter
};

/**
* Normalizes filter content. And splits it by lines.
* @param file Filter content.
*
* Splits filter by lines.
* @param filter Filter to split.
* @returns Array of strings.
*/
const normalizeFilterContent = (file: string): string[] => {
return file
.trim()
.split(/[\r\n]+/);
const splitFilter = (filter: string): string[] => {
return filter.trim().split(/[\r\n]+/);
};

/**
Expand All @@ -631,20 +628,20 @@ const FiltersDownloaderCreator = (FileDownloadWrapper: IFileDownloader): IFilter
? `${filterUrlOrigin}/${url}`
: url;

const file = await FileDownloadWrapper.getExternalFile(filterUrl);
const rawFilter = await FileDownloadWrapper.getExternalFile(filterUrl);

if (downloadOptions && downloadOptions.validateChecksum) {
if (!isValidChecksum(file, downloadOptions.validateChecksumStrict)) {
if (!isValidChecksum(rawFilter, downloadOptions.validateChecksumStrict)) {
throw new Error('Invalid checksum');
}
}

const filter = normalizeFilterContent(file);
const filter = splitFilter(rawFilter);

if (!downloadOptions?.resolveDirectives) {
return {
filter,
rawFilter: filter,
rawFilter,
};
}

Expand All @@ -658,7 +655,7 @@ const FiltersDownloaderCreator = (FileDownloadWrapper: IFileDownloader): IFilter

return {
filter: includesResult,
rawFilter: filter,
rawFilter,
};
};

Expand Down Expand Up @@ -702,20 +699,20 @@ const FiltersDownloaderCreator = (FileDownloadWrapper: IFileDownloader): IFilter
: url;

const origin = getFilterUrlOrigin(urlToLoad, filterOrigin);
const rawFilterContent = await FileDownloadWrapper.getLocalFile(urlToLoad, origin);
const rawFilter = await FileDownloadWrapper.getLocalFile(urlToLoad, origin);

if (downloadOptions && downloadOptions.validateChecksum) {
if (!isValidChecksum(rawFilterContent, downloadOptions.validateChecksumStrict)) {
if (!isValidChecksum(rawFilter, downloadOptions.validateChecksumStrict)) {
throw new Error('Invalid checksum');
}
}

const filterContent = normalizeFilterContent(rawFilterContent);
const filterContent = splitFilter(rawFilter);

if (!downloadOptions?.resolveDirectives) {
return {
filter: filterContent,
rawFilter: filterContent,
rawFilter,
};
}

Expand All @@ -725,7 +722,7 @@ const FiltersDownloaderCreator = (FileDownloadWrapper: IFileDownloader): IFilter
const includesResult = await resolveIncludes(conditionsResult, urlOrigin, downloadOptions.definedExpressions);
return {
filter: includesResult,
rawFilter: filterContent,
rawFilter,
};
};

Expand Down Expand Up @@ -799,11 +796,12 @@ const FiltersDownloaderCreator = (FileDownloadWrapper: IFileDownloader): IFilter
* @returns A Promise that resolves to the result of resolving the includes.
*/
async function resolveConditionsAndIncludes(
rawFilter: string[],
rawFilter: string,
options: DownloadWithRawOptions,
filterUrlOrigin?: string,
): Promise<string[]> {
const resolvedConditionsResult = resolveConditions(rawFilter, options.definedExpressions);
const filter = splitFilter(rawFilter);
const resolvedConditionsResult = resolveConditions(filter, options.definedExpressions);
return resolveIncludes(
resolvedConditionsResult,
filterUrlOrigin,
Expand Down Expand Up @@ -847,15 +845,6 @@ const FiltersDownloaderCreator = (FileDownloadWrapper: IFileDownloader): IFilter
};
}

/**
* Splits filter by lines.
* @param filter Filter to split.
* @returns Array of strings.
*/
const splitFilter = (filter: string): string[] => {
return filter.trim().split(/[\r\n]+/);
};

/**
* Downloads filter rules from a URL without resolving pre-processor directives.
*
Expand Down Expand Up @@ -897,7 +886,7 @@ const FiltersDownloaderCreator = (FileDownloadWrapper: IFileDownloader): IFilter
if (e instanceof UnacceptableResponseError) {
return {
filter: splitFilter(options.rawFilter),
rawFilter: splitFilter(options.rawFilter),
rawFilter: options.rawFilter,
isPatchUpdateFailed: true,
};
}
Expand All @@ -916,19 +905,19 @@ const FiltersDownloaderCreator = (FileDownloadWrapper: IFileDownloader): IFilter
if (rawFilter === options.rawFilter) {
return {
filter: splitFilter(options.rawFilter),
rawFilter: splitFilter(options.rawFilter),
rawFilter: options.rawFilter,
};
}

const resolveResult = await resolveConditionsAndIncludes(
splitFilter(rawFilter),
rawFilter,
options,
filterUrlOrigin,
);

return {
filter: resolveResult,
rawFilter: splitFilter(rawFilter),
rawFilter,
};
};

Expand Down

0 comments on commit a890013

Please sign in to comment.