From 7d0c67444da44dd5818f495ddcd418dafedbf07b Mon Sep 17 00:00:00 2001 From: pratishta Date: Wed, 27 Mar 2024 17:05:07 -0400 Subject: [PATCH 1/9] Radius filter works turning it on and off Remove log statements Remove template testing --- client/app/controllers/query-parameters/show-geography.js | 4 ++++ server/src/project/geometry/geometry.service.ts | 3 +++ server/src/project/project.service.ts | 7 ++++--- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/client/app/controllers/query-parameters/show-geography.js b/client/app/controllers/query-parameters/show-geography.js index 8f9f087e..78a1ab54 100644 --- a/client/app/controllers/query-parameters/show-geography.js +++ b/client/app/controllers/query-parameters/show-geography.js @@ -145,6 +145,10 @@ export const projectParams = new QueryParams({ defaultValue: '', refresh: true, }, + blocks_in_radius: { + defaultValue: [], + refresh: true, + }, dcp_ulurp_nonulurp: { defaultValue: [], refresh: true, diff --git a/server/src/project/geometry/geometry.service.ts b/server/src/project/geometry/geometry.service.ts index 7b3747fc..1a78a1d3 100644 --- a/server/src/project/geometry/geometry.service.ts +++ b/server/src/project/geometry/geometry.service.ts @@ -186,6 +186,9 @@ const QUERY_TEMPLATES = { "dcp_project" ), + blocks_in_radius: (queryParamValue) => + containsString('dcp_validatedblock', [queryParamValue], 'dcp_projectbbl'), + dcp_femafloodzonev: queryParamValue => comparisonOperator( "dcp_femafloodzonev", diff --git a/server/src/project/project.service.ts b/server/src/project/project.service.ts index 4f052a70..c51eccac 100644 --- a/server/src/project/project.service.ts +++ b/server/src/project/project.service.ts @@ -214,6 +214,7 @@ const QUERY_TEMPLATES = { } ) ), + blocks_in_radius: queryParamValue => containsAnyOf("dcp_validatedblock", queryParamValue, { childEntity: "dcp_dcp_project_dcp_projectbbl_project" @@ -232,11 +233,11 @@ export const ALLOWED_FILTERS = [ "dcp_publicstatus", // 'Noticed', 'Filed', 'In Public Review', 'Completed', 'Unknown' "dcp_certifiedreferred", "project_applicant_text", - "block", "distance_from_point", "radius_from_point", "zoning-resolutions", - "dcp_applicability" + "dcp_applicability", + "blocks_in_radius", ]; export const generateFromTemplate = (query, template) => { @@ -249,7 +250,7 @@ export const generateFromTemplate = (query, template) => { function generateProjectsFilterString(query) { // Special handling for 'block' query, which must be explicitly ignored if empty // otherwise, unmapped projects will be excluded from the results - if (!query.block) delete query.block; + if (!Object.keys(query.blocks_in_radius).length) delete query.blocks_in_radius; // optional params // apply only those that appear in the query object From 9c6fe9b9a7fef2015632414247eca7dcb151357b Mon Sep 17 00:00:00 2001 From: David Hochbaum Date: Wed, 3 Apr 2024 15:58:01 -0400 Subject: [PATCH 2/9] Radius filter now uses boro + block instead of just block --- server/src/crm/crm.utilities.ts | 28 ++++++++++++++++++- .../src/project/geometry/geometry.service.ts | 8 ++++-- server/src/project/project.service.ts | 4 ++- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/server/src/crm/crm.utilities.ts b/server/src/crm/crm.utilities.ts index 11f06038..475ee549 100644 --- a/server/src/crm/crm.utilities.ts +++ b/server/src/crm/crm.utilities.ts @@ -122,7 +122,33 @@ export function containsAnyOf(propertyName, strings = [], options?) { return `(${not ? 'not ' : ''}${lambdaQueryPrefix}(${containsQuery}))`; } -export const dateParser = function(key, value) { +export function startsWithString(propertyName, string) { + return `startswith(${propertyName}, '${string}')`; +} + +export function startsWithAnyOf(propertyName, strings = [], options?) { + const { + childEntity = '', + comparisonStrategy = startsWithString, + not = false, + } = options || {}; + + const containsQuery = strings + .map((string, i) => { + // in odata syntax, this character o is a variable for scoping + // logic for related entities. it needs to only appear once. + const lambdaScope = (childEntity && i === 0) ? `${childEntity}:` : ''; + const lambdaScopedProperty = childEntity ? `${childEntity}/${propertyName}` : propertyName; + + return `${lambdaScope}${comparisonStrategy(lambdaScopedProperty, string)}`; + }) + .join(' or '); + const lambdaQueryPrefix = childEntity ? `${childEntity}/any` : ''; + + return `(${not ? 'not ' : ''}${lambdaQueryPrefix}(${containsQuery}))`; +} + +export const dateParser = function (key, value) { if (typeof value === 'string') { // YYYY-MM-DDTHH:mm:ss.sssZ => parsed as UTC // YYYY-MM-DD => parsed as local date diff --git a/server/src/project/geometry/geometry.service.ts b/server/src/project/geometry/geometry.service.ts index 1a78a1d3..a0b60216 100644 --- a/server/src/project/geometry/geometry.service.ts +++ b/server/src/project/geometry/geometry.service.ts @@ -187,7 +187,7 @@ const QUERY_TEMPLATES = { ), blocks_in_radius: (queryParamValue) => - containsString('dcp_validatedblock', [queryParamValue], 'dcp_projectbbl'), + containsString('dcp_bblnumber', [queryParamValue], 'dcp_projectbbl'), dcp_femafloodzonev: queryParamValue => comparisonOperator( @@ -363,8 +363,10 @@ export class GeometryService { const blocks = await this.carto.fetchCarto(distinctBlocks, "json", "post"); // note: DTM stores blocks with the borough - return blocks.map(block => `${block.block.substring(1)}`); - } + + return blocks + .map(block => `${block.block}`); + } // Warning! Returns either null or an Object async getBblsGeometry(bbls = []) { diff --git a/server/src/project/project.service.ts b/server/src/project/project.service.ts index c51eccac..aeb03aca 100644 --- a/server/src/project/project.service.ts +++ b/server/src/project/project.service.ts @@ -30,6 +30,8 @@ import { containsString, equalsAnyOf, containsAnyOf, + startsWithAnyOf, + startsWithString, overwriteCodesWithLabels } from "../crm/crm.utilities"; import { ArtifactService } from "../artifact/artifact.service"; @@ -216,7 +218,7 @@ const QUERY_TEMPLATES = { ), blocks_in_radius: queryParamValue => - containsAnyOf("dcp_validatedblock", queryParamValue, { + startsWithAnyOf("dcp_bblnumber", queryParamValue, { childEntity: "dcp_dcp_project_dcp_projectbbl_project" }) }; From 36a6fb16241b928466bc5f7247023a5fcec10fb2 Mon Sep 17 00:00:00 2001 From: pratishta Date: Thu, 25 Apr 2024 14:16:53 -0400 Subject: [PATCH 3/9] Add acceptance test for radius filter --- client/tests/acceptance/filter-checkbox-test.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/client/tests/acceptance/filter-checkbox-test.js b/client/tests/acceptance/filter-checkbox-test.js index 9f12dc5a..31f4debb 100644 --- a/client/tests/acceptance/filter-checkbox-test.js +++ b/client/tests/acceptance/filter-checkbox-test.js @@ -30,6 +30,15 @@ module('Acceptance | filter checkbox', function(hooks) { assert.equal(currentURL().includes('dcp_femafloodzonea=true'), true); }); + test('User clicks radius filter checkbox', async function(assert) { + server.createList('project', 20); + await visit('/'); + await click('[data-test-filter-section="filter-section-radius-filter"] .switch-paddle'); + + assert.equal(currentURL().includes('distance_from_point'), true); + assert.equal(currentURL().includes('radius_from_point'), true); + }); + test('User clicks community district box, fills in community district name, selects CD', async function(assert) { server.createList('project', 20); await visit('/'); From 86f904c7e29f492dd5382fcac88953fddebb2872 Mon Sep 17 00:00:00 2001 From: pratishta Date: Fri, 26 Apr 2024 13:34:01 -0400 Subject: [PATCH 4/9] Only add blocks_in_radius to query if geometry service returns blocks for filter --- server/src/project/project.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/project/project.service.ts b/server/src/project/project.service.ts index aeb03aca..cd22f5af 100644 --- a/server/src/project/project.service.ts +++ b/server/src/project/project.service.ts @@ -607,14 +607,14 @@ export class ProjectService { const blocks = await this.blocksWithinRadius(query); // adds in the blocks filter for use across various query types - const normalizedQuery = { + const normalizedQuery = blocks ? { blocks_in_radius: blocks, ...query // this information is sent as separate filters but must be represented as one // to work correctly with the query template system. // ...blocks - }; + } : {...query}; const queryObject = generateQueryObject(normalizedQuery); const spatialInfo = await this.geometryService.createAnonymousMapWithFilters( From 41a69e55f4ca5a44237b4770ee3bb8594be4be0d Mon Sep 17 00:00:00 2001 From: pratishta Date: Tue, 30 Apr 2024 09:46:40 -0400 Subject: [PATCH 5/9] Debugging --- server/src/project/geometry/geometry.service.ts | 2 ++ server/src/project/project.service.ts | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/server/src/project/geometry/geometry.service.ts b/server/src/project/geometry/geometry.service.ts index a0b60216..31379bb0 100644 --- a/server/src/project/geometry/geometry.service.ts +++ b/server/src/project/geometry/geometry.service.ts @@ -360,6 +360,8 @@ export class GeometryService { async getBlocksFromRadiusQuery(x, y, radius) { const queryForBlocks = QUERIES.blocksWithinRadius(x, y, radius); const distinctBlocks = `SELECT DISTINCT(block) FROM (${queryForBlocks}) blocksWithinRadius`; + // console.log("distinctBlocks", distinctBlocks); + // console.log("distinctblocks records", await this.carto.fetchCarto(queryForBlocks, "json", "post")) const blocks = await this.carto.fetchCarto(distinctBlocks, "json", "post"); // note: DTM stores blocks with the borough diff --git a/server/src/project/project.service.ts b/server/src/project/project.service.ts index cd22f5af..7c271f7c 100644 --- a/server/src/project/project.service.ts +++ b/server/src/project/project.service.ts @@ -252,8 +252,15 @@ export const generateFromTemplate = (query, template) => { function generateProjectsFilterString(query) { // Special handling for 'block' query, which must be explicitly ignored if empty // otherwise, unmapped projects will be excluded from the results - if (!Object.keys(query.blocks_in_radius).length) delete query.blocks_in_radius; + const radius_filter = query.distance_from_point && query.radius_from_point; + if (!radius_filter && !Object.keys(query.blocks_in_radius).length) { + delete query.blocks_in_radius; + } + + // if (!Object.keys(query.blocks_in_radius).length) + // if (!query.blocks_in_radius) delete query.blocks_in_radius; + console.log("query here ", query); // optional params // apply only those that appear in the query object const requestedFiltersQuery = generateFromTemplate(query, QUERY_TEMPLATES); @@ -605,6 +612,7 @@ export class ProjectService { async queryProjects(query, itemsPerPage = ITEMS_PER_PAGE) { const blocks = await this.blocksWithinRadius(query); + console.log(blocks); // adds in the blocks filter for use across various query types const normalizedQuery = blocks ? { @@ -616,7 +624,12 @@ export class ProjectService { // ...blocks } : {...query}; + console.log("normalizedQuery, ", normalizedQuery); + const queryObject = generateQueryObject(normalizedQuery); + + // console.log("queryObject ", queryObject); + const spatialInfo = await this.geometryService.createAnonymousMapWithFilters( normalizedQuery ); From 34bfa740082b31f32dbc6edc569d3b9977f58c4c Mon Sep 17 00:00:00 2001 From: pratishta Date: Tue, 30 Apr 2024 11:36:12 -0400 Subject: [PATCH 6/9] Check for radius filter and force fake lot if empty block list --- server/src/project/project.service.ts | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/server/src/project/project.service.ts b/server/src/project/project.service.ts index 7c271f7c..b2b93494 100644 --- a/server/src/project/project.service.ts +++ b/server/src/project/project.service.ts @@ -253,14 +253,10 @@ function generateProjectsFilterString(query) { // Special handling for 'block' query, which must be explicitly ignored if empty // otherwise, unmapped projects will be excluded from the results const radius_filter = query.distance_from_point && query.radius_from_point; - if (!radius_filter && !Object.keys(query.blocks_in_radius).length) { - delete query.blocks_in_radius; - } - + if (!Object.keys(query.blocks_in_radius).length) { + radius_filter ? query.blocks_in_radius = ['000000'] : delete query.blocks_in_radius + } - // if (!Object.keys(query.blocks_in_radius).length) - // if (!query.blocks_in_radius) delete query.blocks_in_radius; - console.log("query here ", query); // optional params // apply only those that appear in the query object const requestedFiltersQuery = generateFromTemplate(query, QUERY_TEMPLATES); @@ -615,14 +611,14 @@ export class ProjectService { console.log(blocks); // adds in the blocks filter for use across various query types - const normalizedQuery = blocks ? { + const normalizedQuery = { blocks_in_radius: blocks, ...query // this information is sent as separate filters but must be represented as one // to work correctly with the query template system. // ...blocks - } : {...query}; + }; console.log("normalizedQuery, ", normalizedQuery); From 6ab063c7c790fecc80a2370c323468eb2f4047b4 Mon Sep 17 00:00:00 2001 From: pratishta Date: Tue, 30 Apr 2024 12:06:25 -0400 Subject: [PATCH 7/9] Force fake null block to compare for radius filter --- server/src/project/geometry/geometry.service.ts | 2 -- server/src/project/project.service.ts | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/server/src/project/geometry/geometry.service.ts b/server/src/project/geometry/geometry.service.ts index 31379bb0..a0b60216 100644 --- a/server/src/project/geometry/geometry.service.ts +++ b/server/src/project/geometry/geometry.service.ts @@ -360,8 +360,6 @@ export class GeometryService { async getBlocksFromRadiusQuery(x, y, radius) { const queryForBlocks = QUERIES.blocksWithinRadius(x, y, radius); const distinctBlocks = `SELECT DISTINCT(block) FROM (${queryForBlocks}) blocksWithinRadius`; - // console.log("distinctBlocks", distinctBlocks); - // console.log("distinctblocks records", await this.carto.fetchCarto(queryForBlocks, "json", "post")) const blocks = await this.carto.fetchCarto(distinctBlocks, "json", "post"); // note: DTM stores blocks with the borough diff --git a/server/src/project/project.service.ts b/server/src/project/project.service.ts index b2b93494..bcd848e5 100644 --- a/server/src/project/project.service.ts +++ b/server/src/project/project.service.ts @@ -250,8 +250,8 @@ export const generateFromTemplate = (query, template) => { }; function generateProjectsFilterString(query) { - // Special handling for 'block' query, which must be explicitly ignored if empty - // otherwise, unmapped projects will be excluded from the results + // if blocks_in_radius is empty while the radius_filter is on, force api + // to match a non-existant block to return no projects. Otherwise delete attribute const radius_filter = query.distance_from_point && query.radius_from_point; if (!Object.keys(query.blocks_in_radius).length) { radius_filter ? query.blocks_in_radius = ['000000'] : delete query.blocks_in_radius From 8f49861b6142438160d91ee80ce3d2ab5e4c0d58 Mon Sep 17 00:00:00 2001 From: pratishta Date: Mon, 13 May 2024 14:50:05 -0400 Subject: [PATCH 8/9] Return empty records when no blocks Remove logs --- server/src/project/project.service.ts | 45 +++++++++++---------------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/server/src/project/project.service.ts b/server/src/project/project.service.ts index bcd848e5..ef6c3ed4 100644 --- a/server/src/project/project.service.ts +++ b/server/src/project/project.service.ts @@ -217,7 +217,7 @@ const QUERY_TEMPLATES = { ) ), - blocks_in_radius: queryParamValue => + blocks_in_radius: queryParamValue => startsWithAnyOf("dcp_bblnumber", queryParamValue, { childEntity: "dcp_dcp_project_dcp_projectbbl_project" }) @@ -250,14 +250,6 @@ export const generateFromTemplate = (query, template) => { }; function generateProjectsFilterString(query) { - // if blocks_in_radius is empty while the radius_filter is on, force api - // to match a non-existant block to return no projects. Otherwise delete attribute - const radius_filter = query.distance_from_point && query.radius_from_point; - if (!Object.keys(query.blocks_in_radius).length) { - radius_filter ? query.blocks_in_radius = ['000000'] : delete query.blocks_in_radius - } - - // optional params // apply only those that appear in the query object const requestedFiltersQuery = generateFromTemplate(query, QUERY_TEMPLATES); return all( @@ -269,7 +261,7 @@ function generateProjectsFilterString(query) { ), // optional params ...requestedFiltersQuery - ); + );; } function generateQueryObject(query, overrides?) { @@ -592,7 +584,7 @@ export class ProjectService { async blocksWithinRadius(query) { let { distance_from_point, radius_from_point } = query; - if (!distance_from_point || !radius_from_point) return {}; + if (!distance_from_point || !radius_from_point) return false; // search cannot support more than 1000 because of URI Too Large errors // if (radius_from_point > 1000) radius_from_point = 1000; @@ -608,36 +600,35 @@ export class ProjectService { async queryProjects(query, itemsPerPage = ITEMS_PER_PAGE) { const blocks = await this.blocksWithinRadius(query); - console.log(blocks); // adds in the blocks filter for use across various query types - const normalizedQuery = { + const normalizedQuery = blocks == false ? { ...query } : { blocks_in_radius: blocks, ...query - - // this information is sent as separate filters but must be represented as one - // to work correctly with the query template system. - // ...blocks }; - console.log("normalizedQuery, ", normalizedQuery); - + const empty_radius = blocks == false && normalizedQuery.radius_from_point && normalizedQuery.distance_from_point; const queryObject = generateQueryObject(normalizedQuery); - - // console.log("queryObject ", queryObject); - const spatialInfo = await this.geometryService.createAnonymousMapWithFilters( normalizedQuery ); + + // Empty projects[] when no blocks returned from Carto and radius filter is on const { records: projects, skipTokenParams: nextPageSkipTokenParams, count - } = await this.crmService.queryFromObject( - "dcp_projects", - queryObject, - itemsPerPage - ); + } = empty_radius ? + { + records: [], + skipTokenParams: undefined, + count: 0 + } : + await this.crmService.queryFromObject( + "dcp_projects", + queryObject, + itemsPerPage + ); const valueMappedRecords = overwriteCodesWithLabels( projects, From 3f1bc5cb10ef35b429171245cc67fbb473c3e0fe Mon Sep 17 00:00:00 2001 From: pratishta Date: Wed, 15 May 2024 12:18:49 -0400 Subject: [PATCH 9/9] Refactor blocksWithinRadius and queryProjects methods --- server/src/project/project.service.ts | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/server/src/project/project.service.ts b/server/src/project/project.service.ts index ef6c3ed4..8a456014 100644 --- a/server/src/project/project.service.ts +++ b/server/src/project/project.service.ts @@ -581,11 +581,7 @@ export class ProjectService { return this.serialize(transformedProject); } - async blocksWithinRadius(query) { - let { distance_from_point, radius_from_point } = query; - - if (!distance_from_point || !radius_from_point) return false; - + async blocksWithinRadius(distance_from_point, radius_from_point) { // search cannot support more than 1000 because of URI Too Large errors // if (radius_from_point > 1000) radius_from_point = 1000; @@ -599,26 +595,26 @@ export class ProjectService { } async queryProjects(query, itemsPerPage = ITEMS_PER_PAGE) { - const blocks = await this.blocksWithinRadius(query); - - // adds in the blocks filter for use across various query types - const normalizedQuery = blocks == false ? { ...query } : { + const radiusFilterOn = query.radius_from_point && query.distance_from_point; + const blocks = radiusFilterOn ? await this.blocksWithinRadius(query.distance_from_point, query.radius_from_point) : []; + + const normalizedQuery = blocks.length == 0 ? {...query} : { blocks_in_radius: blocks, ...query }; - const empty_radius = blocks == false && normalizedQuery.radius_from_point && normalizedQuery.distance_from_point; const queryObject = generateQueryObject(normalizedQuery); const spatialInfo = await this.geometryService.createAnonymousMapWithFilters( normalizedQuery ); - // Empty projects[] when no blocks returned from Carto and radius filter is on + // Return empty projects when radius filter is on and Carto returns 0 blocks + // otherwise, send the OData query const { records: projects, skipTokenParams: nextPageSkipTokenParams, count - } = empty_radius ? + } = blocks.length == 0 && radiusFilterOn ? { records: [], skipTokenParams: undefined,