From fd2ceaba7356b6e04f4bd8bfa82cfbc92e4f016a Mon Sep 17 00:00:00 2001 From: Brandon Cohen Date: Wed, 2 Aug 2023 21:44:59 -0400 Subject: [PATCH 1/7] add my script --- Pipfile | 1 + Pipfile.lock | 354 ++++++++++++++++++++++----------------------------- script.py | 205 +++++++++++++++++++++++++++++ 3 files changed, 355 insertions(+), 205 deletions(-) create mode 100644 script.py diff --git a/Pipfile b/Pipfile index aef7b466..7dfbb58d 100644 --- a/Pipfile +++ b/Pipfile @@ -15,6 +15,7 @@ scikit-learn = "*" mapclassify = "*" black = "*" fiona = "*" +esridump = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 34d157b9..7883cc7a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "56fef17634b2e1a6d5d5779ddb0eac1f86a0c153ed0848d2d2e669f1990ad4cb" + "sha256": "b1e540e2ca2141f8538cc433316660fac0aa0a767b4327edec27fbe92da97600" }, "pipfile-spec": 6, "requires": { @@ -33,6 +33,14 @@ "markers": "python_version >= '3.7'", "version": "==3.7.1" }, + "appnope": { + "hashes": [ + "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24", + "sha256:265a455292d0bd8a72453494fa24df5a11eb18373a60c7c0430889f22548605e" + ], + "markers": "platform_system == 'Darwin'", + "version": "==0.1.3" + }, "argon2-cffi": { "hashes": [ "sha256:8c976986f2c5c0e5000919e6de187906cfd81fb1c72bf9d88c01177e77da7f80", @@ -85,11 +93,11 @@ }, "async-lru": { "hashes": [ - "sha256:00c0a8899c20b9c88663a47732689ff98189c9fa08ad9f734d7722f934d250b1", - "sha256:b714c9d1415fca4e264da72a9e2abc66880ce7430e03a973341f88ea4c0d4869" + "sha256:b8a59a5df60805ff63220b2a0c5b5393da5521b113cd5465a44eb037d81a5627", + "sha256:ff02944ce3c288c5be660c42dbcca0742b32c3b279d6dceda655190240b99224" ], "markers": "python_version >= '3.8'", - "version": "==2.0.3" + "version": "==2.0.4" }, "attrs": { "hashes": [ @@ -339,21 +347,13 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'", "version": "==0.7.2" }, - "colorama": { - "hashes": [ - "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", - "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" - ], - "markers": "platform_system == 'Windows'", - "version": "==0.4.6" - }, "comm": { "hashes": [ - "sha256:16613c6211e20223f215fc6d3b266a247b6e2641bf4e0a3ad34cb1aff2aa3f37", - "sha256:a61efa9daffcfbe66fd643ba966f846a624e4e6d6767eda9cf6e993aadaab93e" + "sha256:354e40a59c9dd6db50c5cc6b4acc887d82e9603787f83b68c01a80a923984d15", + "sha256:6d52794cba11b36ed9860999cd10fd02d6b2eac177068fdd585e1e2f8a96e67a" ], "markers": "python_version >= '3.6'", - "version": "==0.1.3" + "version": "==0.1.4" }, "contourpy": { "hashes": [ @@ -410,27 +410,27 @@ }, "debugpy": { "hashes": [ - "sha256:0679b7e1e3523bd7d7869447ec67b59728675aadfc038550a63a362b63029d2c", - "sha256:279d64c408c60431c8ee832dfd9ace7c396984fd7341fa3116aee414e7dcd88d", - "sha256:33edb4afa85c098c24cc361d72ba7c21bb92f501104514d4ffec1fb36e09c01a", - "sha256:38ed626353e7c63f4b11efad659be04c23de2b0d15efff77b60e4740ea685d07", - "sha256:5224eabbbeddcf1943d4e2821876f3e5d7d383f27390b82da5d9558fd4eb30a9", - "sha256:53f7a456bc50706a0eaabecf2d3ce44c4d5010e46dfc65b6b81a518b42866267", - "sha256:9cd10cf338e0907fdcf9eac9087faa30f150ef5445af5a545d307055141dd7a4", - "sha256:aaf6da50377ff4056c8ed470da24632b42e4087bc826845daad7af211e00faad", - "sha256:b3e7ac809b991006ad7f857f016fa92014445085711ef111fdc3f74f66144096", - "sha256:bae1123dff5bfe548ba1683eb972329ba6d646c3a80e6b4c06cd1b1dd0205e9b", - "sha256:c0ff93ae90a03b06d85b2c529eca51ab15457868a377c4cc40a23ab0e4e552a3", - "sha256:c4c2f0810fa25323abfdfa36cbbbb24e5c3b1a42cb762782de64439c575d67f2", - "sha256:d71b31117779d9a90b745720c0eab54ae1da76d5b38c8026c654f4a066b0130a", - "sha256:dbe04e7568aa69361a5b4c47b4493d5680bfa3a911d1e105fbea1b1f23f3eb45", - "sha256:de86029696e1b3b4d0d49076b9eba606c226e33ae312a57a46dca14ff370894d", - "sha256:e3876611d114a18aafef6383695dfc3f1217c98a9168c1aaf1a02b01ec7d8d1e", - "sha256:ed6d5413474e209ba50b1a75b2d9eecf64d41e6e4501977991cdc755dc83ab0f", - "sha256:f90a2d4ad9a035cee7331c06a4cf2245e38bd7c89554fe3b616d90ab8aab89cc" + "sha256:05d1b288167ce3bfc8e1912ebed036207a27b9569ae4476f18287902501689c6", + "sha256:1ca76d3ebb0e6368e107cf2e005e848d3c7705a5b513fdf65470a6f4e49a2de7", + "sha256:1fe3baa28f5a14d8d2a60dded9ea088e27b33f1854ae9a0a1faa1ba03a8b7e47", + "sha256:2345beced3e79fd8ac4158e839a1604d5cccd19beb45561a1ffe2e5b33465f28", + "sha256:3b7091d908dec70022b8966c32b1e9eaf183ff05291edf1d147fee153f4cb9f8", + "sha256:406b3a6cb7548d73260f69a511178ec9196779cafda68e563488c6f94cc88670", + "sha256:4172383b961a2334d29168c7f7b24f2f99d29291a945016986c78a5683fba915", + "sha256:5502e14de6b7241ecf7c4fa4ec6dd61d0824da7a09020c7ffe7be4cd09d36f24", + "sha256:591aac0e69bc75102d9f9294f1228e5d9ff9aa17b8c88e48b1bbb3dab8a54dcc", + "sha256:6830947f68b41cd6abe20941ec3303a8452c40ff5fe3637c6efe233e395ecebc", + "sha256:6ca1c92e30e2aaeca156d5bd76e1587c23e332474a7b12e1900dd632b31ce05e", + "sha256:8c1f5a3286fb633f691c594649e9d2e8e30292c9eaf49e38d7da525151b33a83", + "sha256:959f9b8181a4c544b067daff8d881cd3ac4c7aec1a3a4f41f81c529795b3d864", + "sha256:95f7ce92450b72abcf0c479539a7d00c20e68f1f1fb447eef0b08d2a635d96d7", + "sha256:bb27b8e08f8e60705de6cf05b5da4c21e5a0bc2ca73f06fc36646f456df18ff5", + "sha256:f16bb157b6018ce6a23b64653a6b1892f046cc2b0576df1794c6b22f9fd82118", + "sha256:f4a7193cec3f1e188963f6e8699e1187f758a0a4bbce511b3ad40caf618fc888", + "sha256:f7a80c50b89d8fb49c9e5b6ee28c0bfb822fbd33fef0f2f9843724d0d1984e4e" ], "markers": "python_version >= '3.7'", - "version": "==1.6.7" + "version": "==1.6.8" }, "decorator": { "hashes": [ @@ -448,13 +448,13 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==0.7.1" }, - "exceptiongroup": { + "esridump": { "hashes": [ - "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5", - "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f" + "sha256:3ea0b856da1a55b327375fb1eec24d08ec2a5eeda8700a9e696159252876dca7", + "sha256:bfda87d3b9fc55dcfe1c1df583c98a7117f3861c02443e1350f77ffcca2237a2" ], - "markers": "python_version < '3.11'", - "version": "==1.1.2" + "index": "pypi", + "version": "==1.13.0" }, "executing": { "hashes": [ @@ -498,43 +498,43 @@ }, "fonttools": { "hashes": [ - "sha256:1df1b6f4c7c4bc8201eb47f3b268adbf2539943aa43c400f84556557e3e109c0", - "sha256:2a22b2c425c698dcd5d6b0ff0b566e8e9663172118db6fd5f1941f9b8063da9b", - "sha256:33191f062549e6bb1a4782c22a04ebd37009c09360e2d6686ac5083774d06d95", - "sha256:38cdecd8f1fd4bf4daae7fed1b3170dfc1b523388d6664b2204b351820aa78a7", - "sha256:3ae64303ba670f8959fdaaa30ba0c2dabe75364fdec1caeee596c45d51ca3425", - "sha256:3d1f9471134affc1e3b1b806db6e3e2ad3fa99439e332f1881a474c825101096", - "sha256:4e3334d51f0e37e2c6056e67141b2adabc92613a968797e2571ca8a03bd64773", - "sha256:4edc795533421e98f60acee7d28fc8d941ff5ac10f44668c9c3635ad72ae9045", - "sha256:547ab36a799dded58a46fa647266c24d0ed43a66028cd1cd4370b246ad426cac", - "sha256:59eba8b2e749a1de85760da22333f3d17c42b66e03758855a12a2a542723c6e7", - "sha256:704bccd69b0abb6fab9f5e4d2b75896afa48b427caa2c7988792a2ffce35b441", - "sha256:73ef0bb5d60eb02ba4d3a7d23ada32184bd86007cb2de3657cfcb1175325fc83", - "sha256:7763316111df7b5165529f4183a334aa24c13cdb5375ffa1dc8ce309c8bf4e5c", - "sha256:849ec722bbf7d3501a0e879e57dec1fc54919d31bff3f690af30bb87970f9784", - "sha256:891cfc5a83b0307688f78b9bb446f03a7a1ad981690ac8362f50518bc6153975", - "sha256:952cb405f78734cf6466252fec42e206450d1a6715746013f64df9cbd4f896fa", - "sha256:a7bbb290d13c6dd718ec2c3db46fe6c5f6811e7ea1e07f145fd8468176398224", - "sha256:a9b3cc10dc9e0834b6665fd63ae0c6964c6bc3d7166e9bc84772e0edd09f9fa2", - "sha256:aaaef294d8e411f0ecb778a0aefd11bb5884c9b8333cc1011bdaf3b58ca4bd75", - "sha256:afce2aeb80be72b4da7dd114f10f04873ff512793d13ce0b19d12b2a4c44c0f0", - "sha256:b0938ebbeccf7c80bb9a15e31645cf831572c3a33d5cc69abe436e7000c61b14", - "sha256:b2d1ee95be42b80d1f002d1ee0a51d7a435ea90d36f1a5ae331be9962ee5a3f1", - "sha256:b927e5f466d99c03e6e20961946314b81d6e3490d95865ef88061144d9f62e38", - "sha256:bdd729744ae7ecd7f7311ad25d99da4999003dcfe43b436cf3c333d4e68de73d", - "sha256:c2071267deaa6d93cb16288613419679c77220543551cbe61da02c93d92df72f", - "sha256:cac73bbef7734e78c60949da11c4903ee5837168e58772371bd42a75872f4f82", - "sha256:da2c2964bdc827ba6b8a91dc6de792620be4da3922c4cf0599f36a488c07e2b2", - "sha256:e16a9449f21a93909c5be2f5ed5246420f2316e94195dbfccb5238aaa38f9751", - "sha256:e5c2b0a95a221838991e2f0e455dec1ca3a8cc9cd54febd68cc64d40fdb83669", - "sha256:ec453a45778524f925a8f20fd26a3326f398bfc55d534e37bab470c5e415caa1", - "sha256:edee0900cf0eedb29d17c7876102d6e5a91ee333882b1f5abc83e85b934cadb5", - "sha256:f14f3ccea4cc7dd1b277385adf3c3bf18f9860f87eab9c2fb650b0af16800f55", - "sha256:f240d9adf0583ac8fc1646afe7f4ac039022b6f8fa4f1575a2cfa53675360b69", - "sha256:f48602c0b3fd79cd83a34c40af565fe6db7ac9085c8823b552e6e751e3a5b8be" + "sha256:01cfe02416b6d416c5c8d15e30315cbcd3e97d1b50d3b34b0ce59f742ef55258", + "sha256:0a1466713e54bdbf5521f2f73eebfe727a528905ff5ec63cda40961b4b1eea95", + "sha256:0df8ef75ba5791e873c9eac2262196497525e3f07699a2576d3ab9ddf41cb619", + "sha256:10dac980f2b975ef74532e2a94bb00e97a95b4595fb7f98db493c474d5f54d0e", + "sha256:150122ed93127a26bc3670ebab7e2add1e0983d30927733aec327ebf4255b072", + "sha256:1f81ed9065b4bd3f4f3ce8e4873cd6a6b3f4e92b1eddefde35d332c6f414acc3", + "sha256:27ec3246a088555629f9f0902f7412220c67340553ca91eb540cf247aacb1983", + "sha256:2d6dc3fa91414ff4daa195c05f946e6a575bd214821e26d17ca50f74b35b0fe4", + "sha256:329341ba3d86a36e482610db56b30705384cb23bd595eac8cbb045f627778e9d", + "sha256:3fb2a69870bfe143ec20b039a1c8009e149dd7780dd89554cc8a11f79e5de86b", + "sha256:4655c480a1a4d706152ff54f20e20cf7609084016f1df3851cce67cef768f40a", + "sha256:48e82d776d2e93f88ca56567509d102266e7ab2fb707a0326f032fe657335238", + "sha256:57b68eab183fafac7cd7d464a7bfa0fcd4edf6c67837d14fb09c1c20516cf20b", + "sha256:58c1165f9b2662645de9b19a8c8bdd636b36294ccc07e1b0163856b74f10bafc", + "sha256:614b1283dca88effd20ee48160518e6de275ce9b5456a3134d5f235523fc5065", + "sha256:685a4dd6cf31593b50d6d441feb7781a4a7ef61e19551463e14ed7c527b86f9f", + "sha256:6bd7e4777bff1dcb7c4eff4786998422770f3bfbef8be401c5332895517ba3fa", + "sha256:703101eb0490fae32baf385385d47787b73d9ea55253df43b487c89ec767e0d7", + "sha256:83b98be5d291e08501bd4fc0c4e0f8e6e05b99f3924068b17c5c9972af6fff84", + "sha256:8ece1886d12bb36c48c00b2031518877f41abae317e3a55620d38e307d799b7e", + "sha256:9c456d1f23deff64ffc8b5b098718e149279abdea4d8692dba69172fb6a0d597", + "sha256:9cd2363ea7728496827658682d049ffb2e98525e2247ca64554864a8cc945568", + "sha256:a9b55d2a3b360e0c7fc5bd8badf1503ca1c11dd3a1cd20f2c26787ffa145a9c7", + "sha256:ae7df0ae9ee2f3f7676b0ff6f4ebe48ad0acaeeeaa0b6839d15dbf0709f2c5ef", + "sha256:ae881e484702efdb6cf756462622de81d4414c454edfd950b137e9a7352b3cb9", + "sha256:b8600ae7dce6ec3ddfb201abb98c9d53abbf8064d7ac0c8a0d8925e722ccf2a0", + "sha256:c36c904ce0322df01e590ba814d5d69e084e985d7e4c2869378671d79662a7d4", + "sha256:c8bf88f9e3ce347c716921804ef3a8330cb128284eb6c0b6c4b3574f3c580023", + "sha256:d40673b2e927f7cd0819c6f04489dfbeb337b4a7b10fc633c89bf4f34ecb9620", + "sha256:d54e600a2bcfa5cdaa860237765c01804a03b08404d6affcd92942fa7315ffba", + "sha256:dfe7fa7e607f7e8b58d0c32501a3a7cac148538300626d1b930082c90ae7f6bd", + "sha256:e35bed436726194c5e6e094fdfb423fb7afaa0211199f9d245e59e11118c576c", + "sha256:f0290ea7f9945174bd4dfd66e96149037441eb2008f3649094f056201d99e293", + "sha256:fae4e801b774cc62cecf4a57b1eae4097903fced00c608d9e2bc8f84cd87b54a" ], "markers": "python_version >= '3.8'", - "version": "==4.41.1" + "version": "==4.42.0" }, "fqdn": { "hashes": [ @@ -559,29 +559,13 @@ "markers": "python_version >= '3.5'", "version": "==3.4" }, - "importlib-metadata": { - "hashes": [ - "sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb", - "sha256:dbace7892d8c0c4ac1ad096662232f831d4e64f4c4545bd53016a3e9d4654743" - ], - "markers": "python_version < '3.10'", - "version": "==6.8.0" - }, - "importlib-resources": { - "hashes": [ - "sha256:4cf94875a8368bd89531a756df9a9ebe1f150e0f885030b461237bc7f2d905f2", - "sha256:d952faee11004c045f785bb5636e8f885bed30dc3c940d5d42798a2a4541c185" - ], - "markers": "python_version < '3.10'", - "version": "==6.0.0" - }, "ipykernel": { "hashes": [ - "sha256:29cea0a716b1176d002a61d0b0c851f34536495bc4ef7dd0222c88b41b816123", - "sha256:2f5fffc7ad8f1fd5aadb4e171ba9129d9668dbafa374732cf9511ada52d6547f" + "sha256:e342ce84712861be4b248c4a73472be4702c1b0dd77448bfd6bcfb3af9d5ddf9", + "sha256:f0042e867ac3f6bca1679e6a88cbd6a58ed93a44f9d0866aecde6efe8de76659" ], "markers": "python_version >= '3.8'", - "version": "==6.24.0" + "version": "==6.25.0" }, "ipython": { "hashes": [ @@ -600,11 +584,11 @@ }, "ipywidgets": { "hashes": [ - "sha256:50ace0a8886e9a0d68b980db82f94c25d55d21ff2340ed36f802dd9365e94acf", - "sha256:e0aed0c95a1e55b6a123f64305245578bdc09e52965a34941c2b6a578b8c64a0" + "sha256:6c8396cc7b8c95dfb4e9ab0054f48c002f045e7e5d7ae523f559d64e525a98ab", + "sha256:ce97dd90525b3066fd00094690964e7eac14cf9b7745d35565b5eeac20cce687" ], "markers": "python_version >= '3.7'", - "version": "==8.0.7" + "version": "==8.1.0" }, "isoduration": { "hashes": [ @@ -615,11 +599,11 @@ }, "jedi": { "hashes": [ - "sha256:203c1fd9d969ab8f2119ec0a3342e0b49910045abe6af0a3ae83a5764d54639e", - "sha256:bae794c30d07f6d910d32a7048af09b5a39ed740918da923c6b780790ebac612" + "sha256:bcf9894f1753969cbac8022a8c2eaee06bfa3724e4192470aaffe7eb6272b0c4", + "sha256:cb8ce23fbccff0025e9386b5cf85e892f94c9b822378f8da49970471335ac64e" ], "markers": "python_version >= '3.6'", - "version": "==0.18.2" + "version": "==0.19.0" }, "jinja2": { "hashes": [ @@ -653,11 +637,11 @@ }, "jsonschema": { "hashes": [ - "sha256:971be834317c22daaa9132340a51c01b50910724082c2c1a2ac87eeec153a3fe", - "sha256:fb3642735399fa958c0d2aad7057901554596c63349f4f6b283c493cf692a25d" + "sha256:ce71d2f8c7983ef75a756e568317bf54bc531dc3ad7e66a128eae0d51623d8a3", + "sha256:dc274409c36175aad949c68e5ead0853aaffbe8e88c830ae66bb3c7a1728ad2d" ], "markers": "python_version >= '3.8'", - "version": "==4.18.4" + "version": "==4.18.6" }, "jsonschema-specifications": { "hashes": [ @@ -702,11 +686,11 @@ }, "jupyter-events": { "hashes": [ - "sha256:57a2749f87ba387cd1bfd9b22a0875b889237dbf2edc2121ebb22bde47036c17", - "sha256:9a6e9995f75d1b7146b436ea24d696ce3a35bfa8bfe45e0c33c334c79464d0b3" + "sha256:4753da434c13a37c3f3c89b500afa0c0a6241633441421f6adafe2fb2e2b924e", + "sha256:7be27f54b8388c03eefea123a4f79247c5b9381c49fb1cd48615ee191eb12615" ], - "markers": "python_version >= '3.7'", - "version": "==0.6.3" + "markers": "python_version >= '3.8'", + "version": "==0.7.0" }, "jupyter-lsp": { "hashes": [ @@ -983,27 +967,27 @@ }, "nbconvert": { "hashes": [ - "sha256:25e0cf2b663ee0cd5a90afb6b2f2940bf1abe5cc5bc995b88c8156ca65fa7ede", - "sha256:36d3e7bf32f0c075878176cdeeb645931c994cbed5b747bc7a570ba8cd2321f3" + "sha256:3022adadff3f86578a47fab7c2228bb3ca9c56a24345642a22f917f6168b48fc", + "sha256:4a5996bf5f3cd16aa0431897ba1aa4c64842c2079f434b3dc6b8c4b252ef3355" ], "markers": "python_version >= '3.8'", - "version": "==7.7.2" + "version": "==7.7.3" }, "nbformat": { "hashes": [ - "sha256:3a7f52d040639cbd8a3890218c8b0ffb93211588c57446c90095e32ba5881b5d", - "sha256:b7968ebf4811178a4108ee837eae1442e3f054132100f0359219e9ed1ce3ca45" + "sha256:1c5172d786a41b82bcfd0c23f9e6b6f072e8fb49c39250219e4acfff1efe89e9", + "sha256:5f98b5ba1997dff175e77e0c17d5c10a96eaed2cbd1de3533d1fc35d5e111192" ], "markers": "python_version >= '3.8'", - "version": "==5.9.1" + "version": "==5.9.2" }, "nest-asyncio": { "hashes": [ - "sha256:b9a953fb40dceaa587d109609098db21900182b16440652454a146cffb06e8b8", - "sha256:d267cc1ff794403f7df692964d1d2a3fa9418ffea2a3f6859a439ff482fef290" + "sha256:5301c82941b550b3123a1ea772ba9a1c80bad3a182be8c1a5ae6ad3be57a9657", + "sha256:6a80f7b98f24d9083ed24608977c09dd608d83f91cccc24c9d2cba6d10e01c10" ], "markers": "python_version >= '3.5'", - "version": "==1.5.6" + "version": "==1.5.7" }, "networkx": { "hashes": [ @@ -1015,11 +999,11 @@ }, "notebook": { "hashes": [ - "sha256:38b55e6939df0ba73b53212c3b234e41102f1789e0158606cedaebf00abef6c8", - "sha256:71b4e695e658763a2766613176491854708fb46fbe7664bf5e494deeeab92d60" + "sha256:2e16ad4e63ea89f7efbe212ee7c1693fcfa5ab55ffef75047530f74af4bd926c", + "sha256:35327476042140e8739ff8fcfecdc915658ae72b4db72d6e3b537badcdbf9e35" ], "markers": "python_version >= '3.8'", - "version": "==7.0.0" + "version": "==7.0.1" }, "notebook-shim": { "hashes": [ @@ -1031,34 +1015,34 @@ }, "numpy": { "hashes": [ - "sha256:012097b5b0d00a11070e8f2e261128c44157a8689f7dedcf35576e525893f4fe", - "sha256:0d3fe3dd0506a28493d82dc3cf254be8cd0d26f4008a417385cbf1ae95b54004", - "sha256:0def91f8af6ec4bb94c370e38c575855bf1d0be8a8fbfba42ef9c073faf2cf19", - "sha256:1a180429394f81c7933634ae49b37b472d343cccb5bb0c4a575ac8bbc433722f", - "sha256:1d5d3c68e443c90b38fdf8ef40e60e2538a27548b39b12b73132456847f4b631", - "sha256:20e1266411120a4f16fad8efa8e0454d21d00b8c7cee5b5ccad7565d95eb42dd", - "sha256:247d3ffdd7775bdf191f848be8d49100495114c82c2bd134e8d5d075fb386a1c", - "sha256:35a9527c977b924042170a0887de727cd84ff179e478481404c5dc66b4170009", - "sha256:38eb6548bb91c421261b4805dc44def9ca1a6eef6444ce35ad1669c0f1a3fc5d", - "sha256:3d7abcdd85aea3e6cdddb59af2350c7ab1ed764397f8eec97a038ad244d2d105", - "sha256:41a56b70e8139884eccb2f733c2f7378af06c82304959e174f8e7370af112e09", - "sha256:4a90725800caeaa160732d6b31f3f843ebd45d6b5f3eec9e8cc287e30f2805bf", - "sha256:6b82655dd8efeea69dbf85d00fca40013d7f503212bc5259056244961268b66e", - "sha256:6c6c9261d21e617c6dc5eacba35cb68ec36bb72adcff0dee63f8fbc899362588", - "sha256:77d339465dff3eb33c701430bcb9c325b60354698340229e1dff97745e6b3efa", - "sha256:791f409064d0a69dd20579345d852c59822c6aa087f23b07b1b4e28ff5880fcb", - "sha256:9a3a9f3a61480cc086117b426a8bd86869c213fc4072e606f01c4e4b66eb92bf", - "sha256:c1516db588987450b85595586605742879e50dcce923e8973f79529651545b57", - "sha256:c40571fe966393b212689aa17e32ed905924120737194b5d5c1b20b9ed0fb171", - "sha256:d412c1697c3853c6fc3cb9751b4915859c7afe6a277c2bf00acf287d56c4e625", - "sha256:d5154b1a25ec796b1aee12ac1b22f414f94752c5f94832f14d8d6c9ac40bcca6", - "sha256:d736b75c3f2cb96843a5c7f8d8ccc414768d34b0a75f466c05f3a739b406f10b", - "sha256:e8f6049c4878cb16960fbbfb22105e49d13d752d4d8371b55110941fb3b17800", - "sha256:f76aebc3358ade9eacf9bc2bb8ae589863a4f911611694103af05346637df1b7", - "sha256:fd67b306320dcadea700a8f79b9e671e607f8696e98ec255915c0c6d6b818503" + "sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2", + "sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55", + "sha256:1b9735c27cea5d995496f46a8b1cd7b408b3f34b6d50459d9ac8fe3a20cc17bf", + "sha256:2792d23d62ec51e50ce4d4b7d73de8f67a2fd3ea710dcbc8563a51a03fb07b01", + "sha256:3e0746410e73384e70d286f93abf2520035250aad8c5714240b0492a7302fdca", + "sha256:4c3abc71e8b6edba80a01a52e66d83c5d14433cbcd26a40c329ec7ed09f37901", + "sha256:5883c06bb92f2e6c8181df7b39971a5fb436288db58b5a1c3967702d4278691d", + "sha256:5c97325a0ba6f9d041feb9390924614b60b99209a71a69c876f71052521d42a4", + "sha256:60e7f0f7f6d0eee8364b9a6304c2845b9c491ac706048c7e8cf47b83123b8dbf", + "sha256:76b4115d42a7dfc5d485d358728cdd8719be33cc5ec6ec08632a5d6fca2ed380", + "sha256:7dc869c0c75988e1c693d0e2d5b26034644399dd929bc049db55395b1379e044", + "sha256:834b386f2b8210dca38c71a6e0f4fd6922f7d3fcff935dbe3a570945acb1b545", + "sha256:8b77775f4b7df768967a7c8b3567e309f617dd5e99aeb886fa14dc1a0791141f", + "sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f", + "sha256:b79e513d7aac42ae918db3ad1341a015488530d0bb2a6abcbdd10a3a829ccfd3", + "sha256:bb33d5a1cf360304754913a350edda36d5b8c5331a8237268c48f91253c3a364", + "sha256:bec1e7213c7cb00d67093247f8c4db156fd03075f49876957dca4711306d39c9", + "sha256:c5462d19336db4560041517dbb7759c21d181a67cb01b36ca109b2ae37d32418", + "sha256:c5652ea24d33585ea39eb6a6a15dac87a1206a692719ff45d53c5282e66d4a8f", + "sha256:d7806500e4f5bdd04095e849265e55de20d8cc4b661b038957354327f6d9b295", + "sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3", + "sha256:dfe4a913e29b418d096e696ddd422d8a5d13ffba4ea91f9f60440a3b759b0187", + "sha256:eb942bfb6f84df5ce05dbf4b46673ffed0d3da59f13635ea9b926af3deb76926", + "sha256:f08f2e037bba04e707eebf4bc934f1972a315c883a9e0ebfa8a7756eabf9e357", + "sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760" ], "markers": "python_version >= '3.9'", - "version": "==1.25.1" + "version": "==1.25.2" }, "overrides": { "hashes": [ @@ -1125,11 +1109,19 @@ }, "pathspec": { "hashes": [ - "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687", - "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293" + "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20", + "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3" ], "markers": "python_version >= '3.7'", - "version": "==0.11.1" + "version": "==0.11.2" + }, + "pexpect": { + "hashes": [ + "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937", + "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c" + ], + "markers": "sys_platform != 'win32'", + "version": "==4.8.0" }, "pickleshare": { "hashes": [ @@ -1202,11 +1194,11 @@ }, "platformdirs": { "hashes": [ - "sha256:1b42b450ad933e981d56e59f1b97495428c9bd60698baab9f3eb3d00d5822421", - "sha256:ad8291ae0ae5072f66c16945166cb11c63394c7a3ad1b1bc9828ca3162da8c2f" + "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d", + "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d" ], "markers": "python_version >= '3.7'", - "version": "==3.9.1" + "version": "==3.10.0" }, "prometheus-client": { "hashes": [ @@ -1244,6 +1236,13 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==5.9.5" }, + "ptyprocess": { + "hashes": [ + "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", + "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220" + ], + "version": "==0.7.0" + }, "pure-eval": { "hashes": [ "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350", @@ -1328,37 +1327,6 @@ ], "version": "==2023.3" }, - "pywin32": { - "hashes": [ - "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d", - "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65", - "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e", - "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b", - "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4", - "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040", - "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a", - "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36", - "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8", - "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e", - "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802", - "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a", - "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407", - "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0" - ], - "markers": "sys_platform == 'win32' and platform_python_implementation != 'PyPy'", - "version": "==306" - }, - "pywinpty": { - "hashes": [ - "sha256:0ffd287751ad871141dc9724de70ea21f7fc2ff1af50861e0d232cf70739d8c4", - "sha256:452f10ac9ff8ab9151aa8cea9e491a9612a12250b1899278c6a56bc184afb47f", - "sha256:6701867d42aec1239bc0fedf49a336570eb60eb886e81763db77ea2b6c533cc3", - "sha256:e244cffe29a894876e2cd251306efd0d8d64abd5ada0a46150a4a71c0b9ad5c5", - "sha256:e4e7f023c28ca7aa8e1313e53ba80a4d10171fe27857b7e02f99882dfe3e8638" - ], - "markers": "os_name == 'nt'", - "version": "==2.0.11" - }, "pyyaml": { "hashes": [ "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc", @@ -1836,14 +1804,6 @@ "markers": "python_version >= '3.7'", "version": "==1.2.1" }, - "tomli": { - "hashes": [ - "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", - "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" - ], - "markers": "python_version < '3.11'", - "version": "==2.0.1" - }, "tornado": { "hashes": [ "sha256:05615096845cf50a895026f749195bf0b10b8909f9be672f50b0fe69cba368e4", @@ -1869,14 +1829,6 @@ "markers": "python_version >= '3.7'", "version": "==5.9.0" }, - "typing-extensions": { - "hashes": [ - "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36", - "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2" - ], - "markers": "python_version < '3.10'", - "version": "==4.7.1" - }, "tzdata": { "hashes": [ "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a", @@ -1936,14 +1888,6 @@ ], "markers": "python_version >= '3.7'", "version": "==4.0.8" - }, - "zipp": { - "hashes": [ - "sha256:679e51dd4403591b2d6838a48de3d283f3d188412a9782faadf845f298736ba0", - "sha256:ebc15946aa78bd63458992fc81ec3b6f7b1e92d51c35e6de1c3804e73b799147" - ], - "markers": "python_version < '3.10'", - "version": "==3.16.2" } }, "develop": {} diff --git a/script.py b/script.py new file mode 100644 index 00000000..01212834 --- /dev/null +++ b/script.py @@ -0,0 +1,205 @@ +import requests +import geopandas as gpd +import pandas as pd +import datetime +from esridump.dumper import EsriDumper + + +# Set the CRS to use for all layers +use_crs = "EPSG:2272" + + +class FeatureLayer: + ''' + FeatureLayer is a class to represent a GIS dataset. It can be initialized with a URL to an Esri Feature Service, a SQL query to Carto, or a GeoDataFrame. + ''' + + def __init__(self, name, esri_rest_urls=None, carto_sql_queries=None, gdf=None, crs=use_crs): + self.name = name + self.esri_rest_urls = [esri_rest_urls] if isinstance( + esri_rest_urls, str) else esri_rest_urls + self.carto_sql_queries = [carto_sql_queries] if isinstance( + carto_sql_queries, str) else carto_sql_queries + self.gdf = gdf + self.crs = crs + + inputs = [self.esri_rest_urls, self.carto_sql_queries, self.gdf] + non_none_inputs = [i for i in inputs if i is not None] + + if len(non_none_inputs) != 1: + raise ValueError( + 'Exactly one of esri_rest_urls, carto_sql_queries, or gdf must be provided.') + + if self.esri_rest_urls is not None: + self.type = 'esri' + elif self.carto_sql_queries is not None: + self.type = 'carto' + elif self.gdf is not None: + self.type = 'gdf' + + self.load_data() + + def load_data(self): + print(f'Loading data for {self.name}...') + if self.type == 'gdf': + self.gdf = gdf + else: + try: + if self.type == 'esri': + if self.esri_rest_urls is None: + raise ValueError( + 'Must provide a URL to load data from Esri') + + gdfs = [] + for url in self.esri_rest_urls: + self.dumper = EsriDumper(url) + features = [feature for feature in self.dumper] + + geojson_features = { + "type": "FeatureCollection", "features": features} + gdfs.append(gpd.GeoDataFrame.from_features( + geojson_features, crs=self.crs)) + + self.gdf = pd.concat(gdfs, ignore_index=True) + + elif self.type == 'carto': + if self.carto_sql_queries is None: + raise ValueError( + 'Must provide a SQL query to load data from Carto') + + gdfs = [] + for sql_query in self.carto_sql_queries: + response = requests.get( + "https://phl.carto.com/api/v2/sql", params={"q": sql_query}) + + data = response.json()["rows"] + df = pd.DataFrame(data) + gdf = gpd.GeoDataFrame( + df, geometry=gpd.points_from_xy(df.x, df.y), crs=self.crs) + + gdfs.append(gdf) + self.gdf = pd.concat(gdfs, ignore_index=True) + + except Exception as e: + print(f'Error loading data for {self.name}: {e}') + self.gdf = None + + def spatial_join(self, other_layer, how='left', predicate='intersects'): + ''' + Spatial joins in this script are generally left intersect joins. + They also can create duplicates, so we drop duplicates after the join. + Note: We may want to revisit the duplicates. + ''' + self.gdf = gpd.sjoin(self.gdf, other_layer.gdf, + how=how, predicate=predicate) + self.gdf.drop(columns=['index_right'], inplace=True) + self.gdf.drop_duplicates(inplace=True) + + +''' +Load Vacant Properties Datasets +''' + +vacant_props_layers_to_load = [ + 'https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/Vacant_Indicators_Land/FeatureServer/0/', 'https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/Vacant_Indicators_Bldg/FeatureServer/0/'] + + +vacant_properties = FeatureLayer( + name="Vacant Properties", esri_rest_urls=vacant_props_layers_to_load) + + +# ''' +# Load City Owned Properties +# ''' +city_owned_properties = FeatureLayer( + name='City Owned Properties', esri_rest_urls='https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/LAMAAssets/FeatureServer/0/') + +# Discover which vacant properties are city owned using a spatial join +vacant_properties.spatial_join(city_owned_properties) + +# There are some matches which aren't valid because of imprecise parcel boundaries. Use a match on OPA_ID and OPABRT to remove these. +vacant_properties.gdf = vacant_properties.gdf.loc[~((vacant_properties.gdf['OPABRT'].notnull()) & ( + vacant_properties.gdf['OPA_ID'] != vacant_properties.gdf['OPABRT']))] + +# Note: This removes some entries from the dataset, need to revisit this + + +# ''' +# Load PHS Data +# ''' + +phs_layers_to_load = [ + 'https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/PHS_CommunityLandcare/FeatureServer/0/', 'https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/PHS_PhilaLandCare_Maintenance/FeatureServer/0/' +] + +phs_properties = FeatureLayer( + name="PHS Properties", esri_rest_urls=phs_layers_to_load) + +phs_properties.gdf['COMM_PARTN'] = 'PHS' +phs_properties.gdf = phs_properties.gdf[['COMM_PARTN', 'geometry']] + +vacant_properties.spatial_join(phs_properties) +vacant_properties.gdf['COMM_PARTN'].fillna('None', inplace=True) + + +# ''' +# L&I Data +# ''' + +# # L&I Complaints +# one_year_ago = (datetime.datetime.now() - +# datetime.timedelta(days=365)).strftime("%Y-%m-%d") + +# complaints_sql_query = f"SELECT address, service_request_id, subject, status, service_name, service_code, lat as y, lon as x FROM public_cases_fc WHERE requested_datetime >= '{one_year_ago}'" +# l_and_i_complaints = FeatureLayer( +# name='L&I Complaints', carto_sql_queries=complaints_sql_query, type='carto') + +# # filter for only Status = 'Open' +# l_and_i_complaints.gdf = l_and_i_complaints.gdf[l_and_i_complaints.gdf['status'] == 'Open'] + +# # collapse complaints_gdf by address and concatenate the violationcodetitle values into a list with a semicolon separator +# l_and_i_complaints.gdf = l_and_i_complaints.gdf.groupby('address')['service_name'].apply( +# lambda x: '; '.join([val for val in x if val is not None])).reset_index() + +# # rename the column to 'li_complaints' +# l_and_i_complaints.gdf.rename( +# columns={'service_name': 'li_complaints'}, inplace=True) + +# # L&I Code Violations +# violations_sql_query = f"SELECT parcel_id_num, casenumber, casecreateddate, casetype, casestatus, violationnumber, violationcodetitle, violationstatus, opa_account_num, address, opa_owner, geocode_x as x, geocode_y as y FROM violations WHERE violationdate >= '{one_year_ago}'" +# l_and_i_violations = FeatureLayer( +# name='L&I Code Violations', carto_sql_queries=violations_sql_query, type='carto') + +# # Manipulation of L&I data- not changed much from the original script +# all_violations_count_df = l_and_i_violations.gdf.groupby( +# 'opa_account_num').count().reset_index()[['opa_account_num', 'violationnumber']] +# all_violations_count_df = all_violations_count_df.rename( +# columns={'violationnumber': 'all_violations_past_year'}) +# l_and_i_violations.gdf = l_and_i_violations.gdf[( +# l_and_i_violations.gdf['violationstatus'] == 'OPEN')] + +# open_violations_count_df = l_and_i_violations.gdf.groupby( +# 'opa_account_num').count().reset_index()[['opa_account_num', 'violationnumber']] +# open_violations_count_df = open_violations_count_df.rename( +# columns={'violationnumber': 'open_violations_past_year'}) + +# # join the all_violations_count_df and open_violations_count_df dataframes on opa_account_num +# violations_count_gdf = all_violations_count_df.merge( +# open_violations_count_df, how='left', on='opa_account_num') + +# # replace NaN values with 0 +# violations_count_gdf.fillna(0, inplace=True) + +# # convert the all_violations_past_year and open_violations_past_year columns to integers +# violations_count_gdf['all_violations_past_year'] = violations_count_gdf['all_violations_past_year'].astype( +# int) +# violations_count_gdf['open_violations_past_year'] = violations_count_gdf['open_violations_past_year'].astype( +# int) + +# # collapse violations_gdf by address and concatenate the violationcodetitle values into a list with a semicolon separator +# violations_gdf = l_and_i_violations.gdf.groupby('opa_account_num')['violationcodetitle'].apply( +# lambda x: '; '.join([val for val in x if val is not None])).reset_index() + +# # rename the column to 'li_violations' +# violations_gdf.rename( +# columns={'violationcodetitle': 'li_code_violations'}, inplace=True) From a81b7b20a9aa74bbabd6cb32abd9d3ebf644ee5b Mon Sep 17 00:00:00 2001 From: Brandon Cohen Date: Wed, 2 Aug 2023 21:45:52 -0400 Subject: [PATCH 2/7] black formatting --- script.py | 110 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 66 insertions(+), 44 deletions(-) diff --git a/script.py b/script.py index 01212834..bdd90c51 100644 --- a/script.py +++ b/script.py @@ -10,16 +10,22 @@ class FeatureLayer: - ''' - FeatureLayer is a class to represent a GIS dataset. It can be initialized with a URL to an Esri Feature Service, a SQL query to Carto, or a GeoDataFrame. - ''' + """ + FeatureLayer is a class to represent a GIS dataset. It can be initialized with a URL to an Esri Feature Service, a SQL query to Carto, or a GeoDataFrame. + """ - def __init__(self, name, esri_rest_urls=None, carto_sql_queries=None, gdf=None, crs=use_crs): + def __init__( + self, name, esri_rest_urls=None, carto_sql_queries=None, gdf=None, crs=use_crs + ): self.name = name - self.esri_rest_urls = [esri_rest_urls] if isinstance( - esri_rest_urls, str) else esri_rest_urls - self.carto_sql_queries = [carto_sql_queries] if isinstance( - carto_sql_queries, str) else carto_sql_queries + self.esri_rest_urls = ( + [esri_rest_urls] if isinstance(esri_rest_urls, str) else esri_rest_urls + ) + self.carto_sql_queries = ( + [carto_sql_queries] + if isinstance(carto_sql_queries, str) + else carto_sql_queries + ) self.gdf = gdf self.crs = crs @@ -28,27 +34,27 @@ def __init__(self, name, esri_rest_urls=None, carto_sql_queries=None, gdf=None, if len(non_none_inputs) != 1: raise ValueError( - 'Exactly one of esri_rest_urls, carto_sql_queries, or gdf must be provided.') + "Exactly one of esri_rest_urls, carto_sql_queries, or gdf must be provided." + ) if self.esri_rest_urls is not None: - self.type = 'esri' + self.type = "esri" elif self.carto_sql_queries is not None: - self.type = 'carto' + self.type = "carto" elif self.gdf is not None: - self.type = 'gdf' + self.type = "gdf" self.load_data() def load_data(self): - print(f'Loading data for {self.name}...') - if self.type == 'gdf': + print(f"Loading data for {self.name}...") + if self.type == "gdf": self.gdf = gdf else: try: - if self.type == 'esri': + if self.type == "esri": if self.esri_rest_urls is None: - raise ValueError( - 'Must provide a URL to load data from Esri') + raise ValueError("Must provide a URL to load data from Esri") gdfs = [] for url in self.esri_rest_urls: @@ -56,70 +62,86 @@ def load_data(self): features = [feature for feature in self.dumper] geojson_features = { - "type": "FeatureCollection", "features": features} - gdfs.append(gpd.GeoDataFrame.from_features( - geojson_features, crs=self.crs)) + "type": "FeatureCollection", + "features": features, + } + gdfs.append( + gpd.GeoDataFrame.from_features( + geojson_features, crs=self.crs + ) + ) self.gdf = pd.concat(gdfs, ignore_index=True) - elif self.type == 'carto': + elif self.type == "carto": if self.carto_sql_queries is None: raise ValueError( - 'Must provide a SQL query to load data from Carto') + "Must provide a SQL query to load data from Carto" + ) gdfs = [] for sql_query in self.carto_sql_queries: response = requests.get( - "https://phl.carto.com/api/v2/sql", params={"q": sql_query}) + "https://phl.carto.com/api/v2/sql", params={"q": sql_query} + ) data = response.json()["rows"] df = pd.DataFrame(data) gdf = gpd.GeoDataFrame( - df, geometry=gpd.points_from_xy(df.x, df.y), crs=self.crs) + df, geometry=gpd.points_from_xy(df.x, df.y), crs=self.crs + ) gdfs.append(gdf) self.gdf = pd.concat(gdfs, ignore_index=True) except Exception as e: - print(f'Error loading data for {self.name}: {e}') + print(f"Error loading data for {self.name}: {e}") self.gdf = None - def spatial_join(self, other_layer, how='left', predicate='intersects'): - ''' + def spatial_join(self, other_layer, how="left", predicate="intersects"): + """ Spatial joins in this script are generally left intersect joins. They also can create duplicates, so we drop duplicates after the join. Note: We may want to revisit the duplicates. - ''' - self.gdf = gpd.sjoin(self.gdf, other_layer.gdf, - how=how, predicate=predicate) - self.gdf.drop(columns=['index_right'], inplace=True) + """ + self.gdf = gpd.sjoin(self.gdf, other_layer.gdf, how=how, predicate=predicate) + self.gdf.drop(columns=["index_right"], inplace=True) self.gdf.drop_duplicates(inplace=True) -''' +""" Load Vacant Properties Datasets -''' +""" vacant_props_layers_to_load = [ - 'https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/Vacant_Indicators_Land/FeatureServer/0/', 'https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/Vacant_Indicators_Bldg/FeatureServer/0/'] + "https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/Vacant_Indicators_Land/FeatureServer/0/", + "https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/Vacant_Indicators_Bldg/FeatureServer/0/", +] vacant_properties = FeatureLayer( - name="Vacant Properties", esri_rest_urls=vacant_props_layers_to_load) + name="Vacant Properties", esri_rest_urls=vacant_props_layers_to_load +) # ''' # Load City Owned Properties # ''' city_owned_properties = FeatureLayer( - name='City Owned Properties', esri_rest_urls='https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/LAMAAssets/FeatureServer/0/') + name="City Owned Properties", + esri_rest_urls="https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/LAMAAssets/FeatureServer/0/", +) # Discover which vacant properties are city owned using a spatial join vacant_properties.spatial_join(city_owned_properties) # There are some matches which aren't valid because of imprecise parcel boundaries. Use a match on OPA_ID and OPABRT to remove these. -vacant_properties.gdf = vacant_properties.gdf.loc[~((vacant_properties.gdf['OPABRT'].notnull()) & ( - vacant_properties.gdf['OPA_ID'] != vacant_properties.gdf['OPABRT']))] +vacant_properties.gdf = vacant_properties.gdf.loc[ + ~( + (vacant_properties.gdf["OPABRT"].notnull()) + & (vacant_properties.gdf["OPA_ID"] != vacant_properties.gdf["OPABRT"]) + ) +] # Note: This removes some entries from the dataset, need to revisit this @@ -129,17 +151,17 @@ def spatial_join(self, other_layer, how='left', predicate='intersects'): # ''' phs_layers_to_load = [ - 'https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/PHS_CommunityLandcare/FeatureServer/0/', 'https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/PHS_PhilaLandCare_Maintenance/FeatureServer/0/' + "https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/PHS_CommunityLandcare/FeatureServer/0/", + "https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/PHS_PhilaLandCare_Maintenance/FeatureServer/0/", ] -phs_properties = FeatureLayer( - name="PHS Properties", esri_rest_urls=phs_layers_to_load) +phs_properties = FeatureLayer(name="PHS Properties", esri_rest_urls=phs_layers_to_load) -phs_properties.gdf['COMM_PARTN'] = 'PHS' -phs_properties.gdf = phs_properties.gdf[['COMM_PARTN', 'geometry']] +phs_properties.gdf["COMM_PARTN"] = "PHS" +phs_properties.gdf = phs_properties.gdf[["COMM_PARTN", "geometry"]] vacant_properties.spatial_join(phs_properties) -vacant_properties.gdf['COMM_PARTN'].fillna('None', inplace=True) +vacant_properties.gdf["COMM_PARTN"].fillna("None", inplace=True) # ''' From 521f9c4efd41f09be732767c9635baecf6139620 Mon Sep 17 00:00:00 2001 From: Brandon Cohen Date: Wed, 2 Aug 2023 21:52:18 -0400 Subject: [PATCH 3/7] clean up --- script.py | 75 +++++-------------------------------------------------- 1 file changed, 6 insertions(+), 69 deletions(-) diff --git a/script.py b/script.py index bdd90c51..18fc9c53 100644 --- a/script.py +++ b/script.py @@ -124,9 +124,9 @@ def spatial_join(self, other_layer, how="left", predicate="intersects"): ) -# ''' -# Load City Owned Properties -# ''' +""" +Load City Owned Properties +""" city_owned_properties = FeatureLayer( name="City Owned Properties", esri_rest_urls="https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/LAMAAssets/FeatureServer/0/", @@ -146,9 +146,9 @@ def spatial_join(self, other_layer, how="left", predicate="intersects"): # Note: This removes some entries from the dataset, need to revisit this -# ''' -# Load PHS Data -# ''' +""" +Load PHS Data +""" phs_layers_to_load = [ "https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/PHS_CommunityLandcare/FeatureServer/0/", @@ -162,66 +162,3 @@ def spatial_join(self, other_layer, how="left", predicate="intersects"): vacant_properties.spatial_join(phs_properties) vacant_properties.gdf["COMM_PARTN"].fillna("None", inplace=True) - - -# ''' -# L&I Data -# ''' - -# # L&I Complaints -# one_year_ago = (datetime.datetime.now() - -# datetime.timedelta(days=365)).strftime("%Y-%m-%d") - -# complaints_sql_query = f"SELECT address, service_request_id, subject, status, service_name, service_code, lat as y, lon as x FROM public_cases_fc WHERE requested_datetime >= '{one_year_ago}'" -# l_and_i_complaints = FeatureLayer( -# name='L&I Complaints', carto_sql_queries=complaints_sql_query, type='carto') - -# # filter for only Status = 'Open' -# l_and_i_complaints.gdf = l_and_i_complaints.gdf[l_and_i_complaints.gdf['status'] == 'Open'] - -# # collapse complaints_gdf by address and concatenate the violationcodetitle values into a list with a semicolon separator -# l_and_i_complaints.gdf = l_and_i_complaints.gdf.groupby('address')['service_name'].apply( -# lambda x: '; '.join([val for val in x if val is not None])).reset_index() - -# # rename the column to 'li_complaints' -# l_and_i_complaints.gdf.rename( -# columns={'service_name': 'li_complaints'}, inplace=True) - -# # L&I Code Violations -# violations_sql_query = f"SELECT parcel_id_num, casenumber, casecreateddate, casetype, casestatus, violationnumber, violationcodetitle, violationstatus, opa_account_num, address, opa_owner, geocode_x as x, geocode_y as y FROM violations WHERE violationdate >= '{one_year_ago}'" -# l_and_i_violations = FeatureLayer( -# name='L&I Code Violations', carto_sql_queries=violations_sql_query, type='carto') - -# # Manipulation of L&I data- not changed much from the original script -# all_violations_count_df = l_and_i_violations.gdf.groupby( -# 'opa_account_num').count().reset_index()[['opa_account_num', 'violationnumber']] -# all_violations_count_df = all_violations_count_df.rename( -# columns={'violationnumber': 'all_violations_past_year'}) -# l_and_i_violations.gdf = l_and_i_violations.gdf[( -# l_and_i_violations.gdf['violationstatus'] == 'OPEN')] - -# open_violations_count_df = l_and_i_violations.gdf.groupby( -# 'opa_account_num').count().reset_index()[['opa_account_num', 'violationnumber']] -# open_violations_count_df = open_violations_count_df.rename( -# columns={'violationnumber': 'open_violations_past_year'}) - -# # join the all_violations_count_df and open_violations_count_df dataframes on opa_account_num -# violations_count_gdf = all_violations_count_df.merge( -# open_violations_count_df, how='left', on='opa_account_num') - -# # replace NaN values with 0 -# violations_count_gdf.fillna(0, inplace=True) - -# # convert the all_violations_past_year and open_violations_past_year columns to integers -# violations_count_gdf['all_violations_past_year'] = violations_count_gdf['all_violations_past_year'].astype( -# int) -# violations_count_gdf['open_violations_past_year'] = violations_count_gdf['open_violations_past_year'].astype( -# int) - -# # collapse violations_gdf by address and concatenate the violationcodetitle values into a list with a semicolon separator -# violations_gdf = l_and_i_violations.gdf.groupby('opa_account_num')['violationcodetitle'].apply( -# lambda x: '; '.join([val for val in x if val is not None])).reset_index() - -# # rename the column to 'li_violations' -# violations_gdf.rename( -# columns={'violationcodetitle': 'li_code_violations'}, inplace=True) From 16fc1d0608a64ce7f4f53423b8d8be604eb55346 Mon Sep 17 00:00:00 2001 From: Brandon Cohen Date: Thu, 24 Aug 2023 22:02:10 -0400 Subject: [PATCH 4/7] starting to work --- Pipfile | 3 + Pipfile.lock | 441 ++++++++++++------ README.md | 26 +- .../2023_02_02_vacant_lots_layer.ipynb | 0 constants.py => archived/constants.py | 0 data_utils.py => archived/data_utils.py | 0 process_data.py => archived/process_data.py | 0 notes.md | 1 + src/__init__.py | 0 src/classes/__init__.py | 0 script.py => src/classes/featurelayer.py | 130 +++--- src/config/__init__.py | 0 src/config/config.py | 2 + src/config/psql.py | 6 + src/constants/__init__.py | 0 src/constants/services.py | 24 + src/data_utils/__init__.py | 0 src/data_utils/city_owned_properties.py | 21 + src/data_utils/l_and_i.py | 98 ++++ src/data_utils/phs_properties.py | 16 + src/script.py | 42 ++ 21 files changed, 589 insertions(+), 221 deletions(-) rename 2023_02_02_vacant_lots_layer.ipynb => archived/2023_02_02_vacant_lots_layer.ipynb (100%) rename constants.py => archived/constants.py (100%) rename data_utils.py => archived/data_utils.py (100%) rename process_data.py => archived/process_data.py (100%) create mode 100644 notes.md create mode 100644 src/__init__.py create mode 100644 src/classes/__init__.py rename script.py => src/classes/featurelayer.py (60%) create mode 100644 src/config/__init__.py create mode 100644 src/config/config.py create mode 100644 src/config/psql.py create mode 100644 src/constants/__init__.py create mode 100644 src/constants/services.py create mode 100644 src/data_utils/__init__.py create mode 100644 src/data_utils/city_owned_properties.py create mode 100644 src/data_utils/l_and_i.py create mode 100644 src/data_utils/phs_properties.py create mode 100644 src/script.py diff --git a/Pipfile b/Pipfile index 7dfbb58d..ba475f75 100644 --- a/Pipfile +++ b/Pipfile @@ -16,6 +16,9 @@ mapclassify = "*" black = "*" fiona = "*" esridump = "*" +sqlalchemy = "*" +psycopg2-binary = "*" +geoalchemy2 = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 7883cc7a..f614ef52 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "b1e540e2ca2141f8538cc433316660fac0aa0a767b4327edec27fbe92da97600" + "sha256": "304d3732a0e4467e0533740d3ca4dcd9be9a293b58ad52e9f48d6559f0013bfe" }, "pipfile-spec": 6, "requires": { @@ -43,11 +43,11 @@ }, "argon2-cffi": { "hashes": [ - "sha256:8c976986f2c5c0e5000919e6de187906cfd81fb1c72bf9d88c01177e77da7f80", - "sha256:d384164d944190a7dd7ef22c6aa3ff197da12962bd04b17f64d4e93d934dba5b" + "sha256:879c3e79a2729ce768ebb7d36d4609e3a78a4ca2ec3a9f12286ca057e3d0db08", + "sha256:c670642b78ba29641818ab2e68bd4e6a78ba53b7eff7b4c3815ae16abf91c7ea" ], - "markers": "python_version >= '3.6'", - "version": "==21.3.0" + "markers": "python_version >= '3.7'", + "version": "==23.1.0" }, "argon2-cffi-bindings": { "hashes": [ @@ -410,27 +410,27 @@ }, "debugpy": { "hashes": [ - "sha256:05d1b288167ce3bfc8e1912ebed036207a27b9569ae4476f18287902501689c6", - "sha256:1ca76d3ebb0e6368e107cf2e005e848d3c7705a5b513fdf65470a6f4e49a2de7", - "sha256:1fe3baa28f5a14d8d2a60dded9ea088e27b33f1854ae9a0a1faa1ba03a8b7e47", - "sha256:2345beced3e79fd8ac4158e839a1604d5cccd19beb45561a1ffe2e5b33465f28", - "sha256:3b7091d908dec70022b8966c32b1e9eaf183ff05291edf1d147fee153f4cb9f8", - "sha256:406b3a6cb7548d73260f69a511178ec9196779cafda68e563488c6f94cc88670", - "sha256:4172383b961a2334d29168c7f7b24f2f99d29291a945016986c78a5683fba915", - "sha256:5502e14de6b7241ecf7c4fa4ec6dd61d0824da7a09020c7ffe7be4cd09d36f24", - "sha256:591aac0e69bc75102d9f9294f1228e5d9ff9aa17b8c88e48b1bbb3dab8a54dcc", - "sha256:6830947f68b41cd6abe20941ec3303a8452c40ff5fe3637c6efe233e395ecebc", - "sha256:6ca1c92e30e2aaeca156d5bd76e1587c23e332474a7b12e1900dd632b31ce05e", - "sha256:8c1f5a3286fb633f691c594649e9d2e8e30292c9eaf49e38d7da525151b33a83", - "sha256:959f9b8181a4c544b067daff8d881cd3ac4c7aec1a3a4f41f81c529795b3d864", - "sha256:95f7ce92450b72abcf0c479539a7d00c20e68f1f1fb447eef0b08d2a635d96d7", - "sha256:bb27b8e08f8e60705de6cf05b5da4c21e5a0bc2ca73f06fc36646f456df18ff5", - "sha256:f16bb157b6018ce6a23b64653a6b1892f046cc2b0576df1794c6b22f9fd82118", - "sha256:f4a7193cec3f1e188963f6e8699e1187f758a0a4bbce511b3ad40caf618fc888", - "sha256:f7a80c50b89d8fb49c9e5b6ee28c0bfb822fbd33fef0f2f9843724d0d1984e4e" + "sha256:038c51268367c9c935905a90b1c2d2dbfe304037c27ba9d19fe7409f8cdc710c", + "sha256:1093a5c541af079c13ac8c70ab8b24d1d35c8cacb676306cf11e57f699c02926", + "sha256:3370ef1b9951d15799ef7af41f8174194f3482ee689988379763ef61a5456426", + "sha256:38651c3639a4e8bbf0ca7e52d799f6abd07d622a193c406be375da4d510d968d", + "sha256:3de5d0f97c425dc49bce4293df6a04494309eedadd2b52c22e58d95107e178d9", + "sha256:4b9eba71c290852f959d2cf8a03af28afd3ca639ad374d393d53d367f7f685b2", + "sha256:65b28435a17cba4c09e739621173ff90c515f7b9e8ea469b92e3c28ef8e5cdfb", + "sha256:72f5d2ecead8125cf669e62784ef1e6300f4067b0f14d9f95ee00ae06fc7c4f7", + "sha256:85969d864c45f70c3996067cfa76a319bae749b04171f2cdeceebe4add316155", + "sha256:890f7ab9a683886a0f185786ffbda3b46495c4b929dab083b8c79d6825832a52", + "sha256:903bd61d5eb433b6c25b48eae5e23821d4c1a19e25c9610205f5aeaccae64e32", + "sha256:92b6dae8bfbd497c90596bbb69089acf7954164aea3228a99d7e43e5267f5b36", + "sha256:973a97ed3b434eab0f792719a484566c35328196540676685c975651266fccf9", + "sha256:d16882030860081e7dd5aa619f30dec3c2f9a421e69861125f83cc372c94e57d", + "sha256:d4ac7a4dba28801d184b7fc0e024da2635ca87d8b0a825c6087bb5168e3c0d28", + "sha256:eea8d8cfb9965ac41b99a61f8e755a8f50e9a20330938ad8271530210f54e09c", + "sha256:f0851403030f3975d6e2eaa4abf73232ab90b98f041e3c09ba33be2beda43fcf", + "sha256:fe87ec0182ef624855d05e6ed7e0b7cb1359d2ffa2a925f8ec2d22e98b75d0ca" ], "markers": "python_version >= '3.7'", - "version": "==1.6.8" + "version": "==1.6.7.post1" }, "decorator": { "hashes": [ @@ -543,6 +543,14 @@ ], "version": "==1.5.1" }, + "geoalchemy2": { + "hashes": [ + "sha256:0830c98f83d6b1706e62b5544793d304e2853493d6e70ac18444c13748c3d1c7", + "sha256:620b31cbf97a368b2486dbcfcd36da2081827e933d4163bcb942043b79b545e8" + ], + "index": "pypi", + "version": "==0.14.1" + }, "geopandas": { "hashes": [ "sha256:101cfd0de54bcf9e287a55b5ea17ebe0db53a5e25a28bacf100143d0507cabd9", @@ -561,11 +569,11 @@ }, "ipykernel": { "hashes": [ - "sha256:e342ce84712861be4b248c4a73472be4702c1b0dd77448bfd6bcfb3af9d5ddf9", - "sha256:f0042e867ac3f6bca1679e6a88cbd6a58ed93a44f9d0866aecde6efe8de76659" + "sha256:050391364c0977e768e354bdb60cbbfbee7cbb943b1af1618382021136ffd42f", + "sha256:c8a2430b357073b37c76c21c52184db42f6b4b0e438e1eb7df3c4440d120497c" ], "markers": "python_version >= '3.8'", - "version": "==6.25.0" + "version": "==6.25.1" }, "ipython": { "hashes": [ @@ -615,11 +623,11 @@ }, "joblib": { "hashes": [ - "sha256:1f937906df65329ba98013dc9692fe22a4c5e4a648112de500508b18a21b41e3", - "sha256:89cf0529520e01b3de7ac7b74a8102c90d16d54c64b5dd98cafcd14307fdf915" + "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1", + "sha256:ef4331c65f239985f3f2220ecc87db222f08fd22097a3dd5698f693875f8cbb9" ], "markers": "python_version >= '3.7'", - "version": "==1.3.1" + "version": "==1.3.2" }, "json5": { "hashes": [ @@ -637,11 +645,11 @@ }, "jsonschema": { "hashes": [ - "sha256:ce71d2f8c7983ef75a756e568317bf54bc531dc3ad7e66a128eae0d51623d8a3", - "sha256:dc274409c36175aad949c68e5ead0853aaffbe8e88c830ae66bb3c7a1728ad2d" + "sha256:043dc26a3845ff09d20e4420d6012a9c91c9aa8999fa184e7efcfeccb41e32cb", + "sha256:6e1e7569ac13be8139b2dd2c21a55d350066ee3f80df06c608b398cdc6f30e8f" ], "markers": "python_version >= '3.8'", - "version": "==4.18.6" + "version": "==4.19.0" }, "jsonschema-specifications": { "hashes": [ @@ -702,11 +710,11 @@ }, "jupyter-server": { "hashes": [ - "sha256:36da0a266d31a41ac335a366c88933c17dfa5bb817a48f5c02c16d303bc9477f", - "sha256:6a77912aff643e53fa14bdb2634884b52b784a4be77ce8e93f7283faed0f0849" + "sha256:59838bf20759354e2b4222fed702948a934cb21a503e21cd5b03706a456391d9", + "sha256:76b4ae0b568c331acc9aa904fc4a1194ef9bdefa69f232deb3f3cae802528c05" ], "markers": "python_version >= '3.8'", - "version": "==2.7.0" + "version": "==2.7.1" }, "jupyter-server-terminals": { "hashes": [ @@ -718,11 +726,11 @@ }, "jupyterlab": { "hashes": [ - "sha256:d369944391b1d15f2d1f3cb965fb67352956279b2ae6f03ce7947a43940a8301", - "sha256:e14d1ce46a613028111d0d476a1d7d6b094003b7462bac669f5b478317abcb39" + "sha256:13b3a326e7b95d72746fe20dbe80ee1e71165d6905e01ceaf1320eb809cb1b47", + "sha256:de49deb75f9b9aec478ed04754cbefe9c5d22fd796a5783cdc65e212983d3611" ], "markers": "python_version >= '3.8'", - "version": "==4.0.3" + "version": "==4.0.5" }, "jupyterlab-pygments": { "hashes": [ @@ -967,11 +975,11 @@ }, "nbconvert": { "hashes": [ - "sha256:3022adadff3f86578a47fab7c2228bb3ca9c56a24345642a22f917f6168b48fc", - "sha256:4a5996bf5f3cd16aa0431897ba1aa4c64842c2079f434b3dc6b8c4b252ef3355" + "sha256:1113d039fa3fc3a846ffa5a3b0a019e85aaa94c566a09fa0c400fb7638e46087", + "sha256:ace26f4386d08eb5c55833596a942048c5502a95e05590cb523826a749a40a37" ], "markers": "python_version >= '3.8'", - "version": "==7.7.3" + "version": "==7.7.4" }, "nbformat": { "hashes": [ @@ -999,11 +1007,11 @@ }, "notebook": { "hashes": [ - "sha256:2e16ad4e63ea89f7efbe212ee7c1693fcfa5ab55ffef75047530f74af4bd926c", - "sha256:35327476042140e8739ff8fcfecdc915658ae72b4db72d6e3b537badcdbf9e35" + "sha256:c77b1499dc9b07ce4f4f26990dcb25b2107b434f2536766b51a72a4228d9a4b6", + "sha256:d70d6a07418c829bd5f54337ce993b7105261d9026f9d3fe68e9b8aa1a20da9a" ], "markers": "python_version >= '3.8'", - "version": "==7.0.1" + "version": "==7.0.2" }, "notebook-shim": { "hashes": [ @@ -1046,11 +1054,11 @@ }, "overrides": { "hashes": [ - "sha256:6187d8710a935d09b0bcef8238301d6ee2569d2ac1ae0ec39a8c7924e27f58ca", - "sha256:8b97c6c1e1681b78cbc9424b138d880f0803c2254c5ebaabdde57bb6c62093f2" + "sha256:3ad24583f86d6d7a49049695efe9933e67ba62f0c7625d53c59fa832ce4b8b7d", + "sha256:9502a3cca51f4fac40b5feca985b6703a5c1f6ad815588a7ca9e285b9dca6757" ], "markers": "python_version >= '3.6'", - "version": "==7.3.1" + "version": "==7.4.0" }, "packaging": { "hashes": [ @@ -1236,6 +1244,72 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==5.9.5" }, + "psycopg2-binary": { + "hashes": [ + "sha256:00d8db270afb76f48a499f7bb8fa70297e66da67288471ca873db88382850bf4", + "sha256:024eaeb2a08c9a65cd5f94b31ace1ee3bb3f978cd4d079406aef85169ba01f08", + "sha256:094af2e77a1976efd4956a031028774b827029729725e136514aae3cdf49b87b", + "sha256:1011eeb0c51e5b9ea1016f0f45fa23aca63966a4c0afcf0340ccabe85a9f65bd", + "sha256:11abdbfc6f7f7dea4a524b5f4117369b0d757725798f1593796be6ece20266cb", + "sha256:122641b7fab18ef76b18860dd0c772290566b6fb30cc08e923ad73d17461dc63", + "sha256:17cc17a70dfb295a240db7f65b6d8153c3d81efb145d76da1e4a096e9c5c0e63", + "sha256:18f12632ab516c47c1ac4841a78fddea6508a8284c7cf0f292cb1a523f2e2379", + "sha256:1b918f64a51ffe19cd2e230b3240ba481330ce1d4b7875ae67305bd1d37b041c", + "sha256:1c31c2606ac500dbd26381145684d87730a2fac9a62ebcfbaa2b119f8d6c19f4", + "sha256:26484e913d472ecb6b45937ea55ce29c57c662066d222fb0fbdc1fab457f18c5", + "sha256:2993ccb2b7e80844d534e55e0f12534c2871952f78e0da33c35e648bf002bbff", + "sha256:2b04da24cbde33292ad34a40db9832a80ad12de26486ffeda883413c9e1b1d5e", + "sha256:2dec5a75a3a5d42b120e88e6ed3e3b37b46459202bb8e36cd67591b6e5feebc1", + "sha256:2df562bb2e4e00ee064779902d721223cfa9f8f58e7e52318c97d139cf7f012d", + "sha256:3fbb1184c7e9d28d67671992970718c05af5f77fc88e26fd7136613c4ece1f89", + "sha256:42a62ef0e5abb55bf6ffb050eb2b0fcd767261fa3faf943a4267539168807522", + "sha256:4ecc15666f16f97709106d87284c136cdc82647e1c3f8392a672616aed3c7151", + "sha256:4eec5d36dbcfc076caab61a2114c12094c0b7027d57e9e4387b634e8ab36fd44", + "sha256:4fe13712357d802080cfccbf8c6266a3121dc0e27e2144819029095ccf708372", + "sha256:51d1b42d44f4ffb93188f9b39e6d1c82aa758fdb8d9de65e1ddfe7a7d250d7ad", + "sha256:59f7e9109a59dfa31efa022e94a244736ae401526682de504e87bd11ce870c22", + "sha256:62cb6de84d7767164a87ca97e22e5e0a134856ebcb08f21b621c6125baf61f16", + "sha256:642df77484b2dcaf87d4237792246d8068653f9e0f5c025e2c692fc56b0dda70", + "sha256:6822c9c63308d650db201ba22fe6648bd6786ca6d14fdaf273b17e15608d0852", + "sha256:692df8763b71d42eb8343f54091368f6f6c9cfc56dc391858cdb3c3ef1e3e584", + "sha256:6d92e139ca388ccfe8c04aacc163756e55ba4c623c6ba13d5d1595ed97523e4b", + "sha256:7952807f95c8eba6a8ccb14e00bf170bb700cafcec3924d565235dffc7dc4ae8", + "sha256:7db7b9b701974c96a88997d458b38ccb110eba8f805d4b4f74944aac48639b42", + "sha256:81d5dd2dd9ab78d31a451e357315f201d976c131ca7d43870a0e8063b6b7a1ec", + "sha256:8a136c8aaf6615653450817a7abe0fc01e4ea720ae41dfb2823eccae4b9062a3", + "sha256:8a7968fd20bd550431837656872c19575b687f3f6f98120046228e451e4064df", + "sha256:8c721ee464e45ecf609ff8c0a555018764974114f671815a0a7152aedb9f3343", + "sha256:8f309b77a7c716e6ed9891b9b42953c3ff7d533dc548c1e33fddc73d2f5e21f9", + "sha256:8f94cb12150d57ea433e3e02aabd072205648e86f1d5a0a692d60242f7809b15", + "sha256:95a7a747bdc3b010bb6a980f053233e7610276d55f3ca506afff4ad7749ab58a", + "sha256:9b0c2b466b2f4d89ccc33784c4ebb1627989bd84a39b79092e560e937a11d4ac", + "sha256:9dcfd5d37e027ec393a303cc0a216be564b96c80ba532f3d1e0d2b5e5e4b1e6e", + "sha256:a5ee89587696d808c9a00876065d725d4ae606f5f7853b961cdbc348b0f7c9a1", + "sha256:a6a8b575ac45af1eaccbbcdcf710ab984fd50af048fe130672377f78aaff6fc1", + "sha256:ac83ab05e25354dad798401babaa6daa9577462136ba215694865394840e31f8", + "sha256:ad26d4eeaa0d722b25814cce97335ecf1b707630258f14ac4d2ed3d1d8415265", + "sha256:ad5ec10b53cbb57e9a2e77b67e4e4368df56b54d6b00cc86398578f1c635f329", + "sha256:c82986635a16fb1fa15cd5436035c88bc65c3d5ced1cfaac7f357ee9e9deddd4", + "sha256:ced63c054bdaf0298f62681d5dcae3afe60cbae332390bfb1acf0e23dcd25fc8", + "sha256:d0b16e5bb0ab78583f0ed7ab16378a0f8a89a27256bb5560402749dbe8a164d7", + "sha256:dbbc3c5d15ed76b0d9db7753c0db40899136ecfe97d50cbde918f630c5eb857a", + "sha256:ded8e15f7550db9e75c60b3d9fcbc7737fea258a0f10032cdb7edc26c2a671fd", + "sha256:e02bc4f2966475a7393bd0f098e1165d470d3fa816264054359ed4f10f6914ea", + "sha256:e5666632ba2b0d9757b38fc17337d84bdf932d38563c5234f5f8c54fd01349c9", + "sha256:ea5f8ee87f1eddc818fc04649d952c526db4426d26bab16efbe5a0c52b27d6ab", + "sha256:eb1c0e682138f9067a58fc3c9a9bf1c83d8e08cfbee380d858e63196466d5c86", + "sha256:eb3b8d55924a6058a26db69fb1d3e7e32695ff8b491835ba9f479537e14dcf9f", + "sha256:ee919b676da28f78f91b464fb3e12238bd7474483352a59c8a16c39dfc59f0c5", + "sha256:f02f4a72cc3ab2565c6d9720f0343cb840fb2dc01a2e9ecb8bc58ccf95dc5c06", + "sha256:f4f37bbc6588d402980ffbd1f3338c871368fb4b1cfa091debe13c68bb3852b3", + "sha256:f8651cf1f144f9ee0fa7d1a1df61a9184ab72962531ca99f077bbdcba3947c58", + "sha256:f955aa50d7d5220fcb6e38f69ea126eafecd812d96aeed5d5f3597f33fad43bb", + "sha256:fc10da7e7df3380426521e8c1ed975d22df678639da2ed0ec3244c3dc2ab54c8", + "sha256:fdca0511458d26cf39b827a663d7d87db6f32b93efc22442a742035728603d5f" + ], + "index": "pypi", + "version": "==2.9.7" + }, "ptyprocess": { "hashes": [ "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", @@ -1259,11 +1333,11 @@ }, "pygments": { "hashes": [ - "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c", - "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1" + "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692", + "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29" ], "markers": "python_version >= '3.7'", - "version": "==2.15.1" + "version": "==2.16.1" }, "pyparsing": { "hashes": [ @@ -1375,86 +1449,102 @@ }, "pyzmq": { "hashes": [ - "sha256:01f06f33e12497dca86353c354461f75275a5ad9eaea181ac0dc1662da8074fa", - "sha256:0b6b42f7055bbc562f63f3df3b63e3dd1ebe9727ff0f124c3aa7bcea7b3a00f9", - "sha256:0c4fc2741e0513b5d5a12fe200d6785bbcc621f6f2278893a9ca7bed7f2efb7d", - "sha256:108c96ebbd573d929740d66e4c3d1bdf31d5cde003b8dc7811a3c8c5b0fc173b", - "sha256:13bbe36da3f8aaf2b7ec12696253c0bf6ffe05f4507985a8844a1081db6ec22d", - "sha256:154bddda2a351161474b36dba03bf1463377ec226a13458725183e508840df89", - "sha256:19d0383b1f18411d137d891cab567de9afa609b214de68b86e20173dc624c101", - "sha256:1a6169e69034eaa06823da6a93a7739ff38716142b3596c180363dee729d713d", - "sha256:1fc56a0221bdf67cfa94ef2d6ce5513a3d209c3dfd21fed4d4e87eca1822e3a3", - "sha256:2a21fec5c3cea45421a19ccbe6250c82f97af4175bc09de4d6dd78fb0cb4c200", - "sha256:2b15247c49d8cbea695b321ae5478d47cffd496a2ec5ef47131a9e79ddd7e46c", - "sha256:2f5efcc29056dfe95e9c9db0dfbb12b62db9c4ad302f812931b6d21dd04a9119", - "sha256:2f666ae327a6899ff560d741681fdcdf4506f990595201ed39b44278c471ad98", - "sha256:332616f95eb400492103ab9d542b69d5f0ff628b23129a4bc0a2fd48da6e4e0b", - "sha256:33d5c8391a34d56224bccf74f458d82fc6e24b3213fc68165c98b708c7a69325", - "sha256:3575699d7fd7c9b2108bc1c6128641a9a825a58577775ada26c02eb29e09c517", - "sha256:3830be8826639d801de9053cf86350ed6742c4321ba4236e4b5568528d7bfed7", - "sha256:3a522510e3434e12aff80187144c6df556bb06fe6b9d01b2ecfbd2b5bfa5c60c", - "sha256:3bed53f7218490c68f0e82a29c92335daa9606216e51c64f37b48eb78f1281f4", - "sha256:414b8beec76521358b49170db7b9967d6974bdfc3297f47f7d23edec37329b00", - "sha256:442d3efc77ca4d35bee3547a8e08e8d4bb88dadb54a8377014938ba98d2e074a", - "sha256:47b915ba666c51391836d7ed9a745926b22c434efa76c119f77bcffa64d2c50c", - "sha256:48e5e59e77c1a83162ab3c163fc01cd2eebc5b34560341a67421b09be0891287", - "sha256:4a82faae00d1eed4809c2f18b37f15ce39a10a1c58fe48b60ad02875d6e13d80", - "sha256:4a983c8694667fd76d793ada77fd36c8317e76aa66eec75be2653cef2ea72883", - "sha256:4c2fc7aad520a97d64ffc98190fce6b64152bde57a10c704b337082679e74f67", - "sha256:4cb27ef9d3bdc0c195b2dc54fcb8720e18b741624686a81942e14c8b67cc61a6", - "sha256:4d67609b37204acad3d566bb7391e0ecc25ef8bae22ff72ebe2ad7ffb7847158", - "sha256:5482f08d2c3c42b920e8771ae8932fbaa0a67dff925fc476996ddd8155a170f3", - "sha256:5489738a692bc7ee9a0a7765979c8a572520d616d12d949eaffc6e061b82b4d1", - "sha256:5693dcc4f163481cf79e98cf2d7995c60e43809e325b77a7748d8024b1b7bcba", - "sha256:58416db767787aedbfd57116714aad6c9ce57215ffa1c3758a52403f7c68cff5", - "sha256:5873d6a60b778848ce23b6c0ac26c39e48969823882f607516b91fb323ce80e5", - "sha256:5af31493663cf76dd36b00dafbc839e83bbca8a0662931e11816d75f36155897", - "sha256:5e7fbcafa3ea16d1de1f213c226005fea21ee16ed56134b75b2dede5a2129e62", - "sha256:65346f507a815a731092421d0d7d60ed551a80d9b75e8b684307d435a5597425", - "sha256:6581e886aec3135964a302a0f5eb68f964869b9efd1dbafdebceaaf2934f8a68", - "sha256:69511d604368f3dc58d4be1b0bad99b61ee92b44afe1cd9b7bd8c5e34ea8248a", - "sha256:7018289b402ebf2b2c06992813523de61d4ce17bd514c4339d8f27a6f6809492", - "sha256:71c7b5896e40720d30cd77a81e62b433b981005bbff0cb2f739e0f8d059b5d99", - "sha256:75217e83faea9edbc29516fc90c817bc40c6b21a5771ecb53e868e45594826b0", - "sha256:7e23a8c3b6c06de40bdb9e06288180d630b562db8ac199e8cc535af81f90e64b", - "sha256:80c41023465d36280e801564a69cbfce8ae85ff79b080e1913f6e90481fb8957", - "sha256:831ba20b660b39e39e5ac8603e8193f8fce1ee03a42c84ade89c36a251449d80", - "sha256:851fb2fe14036cfc1960d806628b80276af5424db09fe5c91c726890c8e6d943", - "sha256:8751f9c1442624da391bbd92bd4b072def6d7702a9390e4479f45c182392ff78", - "sha256:8b45d722046fea5a5694cba5d86f21f78f0052b40a4bbbbf60128ac55bfcc7b6", - "sha256:8b697774ea8273e3c0460cf0bba16cd85ca6c46dfe8b303211816d68c492e132", - "sha256:90146ab578931e0e2826ee39d0c948d0ea72734378f1898939d18bc9c823fcf9", - "sha256:9301cf1d7fc1ddf668d0abbe3e227fc9ab15bc036a31c247276012abb921b5ff", - "sha256:95bd3a998d8c68b76679f6b18f520904af5204f089beebb7b0301d97704634dd", - "sha256:968b0c737797c1809ec602e082cb63e9824ff2329275336bb88bd71591e94a90", - "sha256:97d984b1b2f574bc1bb58296d3c0b64b10e95e7026f8716ed6c0b86d4679843f", - "sha256:9e68ae9864d260b18f311b68d29134d8776d82e7f5d75ce898b40a88df9db30f", - "sha256:adecf6d02b1beab8d7c04bc36f22bb0e4c65a35eb0b4750b91693631d4081c70", - "sha256:af56229ea6527a849ac9fb154a059d7e32e77a8cba27e3e62a1e38d8808cb1a5", - "sha256:b324fa769577fc2c8f5efcd429cef5acbc17d63fe15ed16d6dcbac2c5eb00849", - "sha256:b5a07c4f29bf7cb0164664ef87e4aa25435dcc1f818d29842118b0ac1eb8e2b5", - "sha256:bad172aba822444b32eae54c2d5ab18cd7dee9814fd5c7ed026603b8cae2d05f", - "sha256:bdca18b94c404af6ae5533cd1bc310c4931f7ac97c148bbfd2cd4bdd62b96253", - "sha256:be24a5867b8e3b9dd5c241de359a9a5217698ff616ac2daa47713ba2ebe30ad1", - "sha256:be86a26415a8b6af02cd8d782e3a9ae3872140a057f1cadf0133de685185c02b", - "sha256:c66b7ff2527e18554030319b1376d81560ca0742c6e0b17ff1ee96624a5f1afd", - "sha256:c8398a1b1951aaa330269c35335ae69744be166e67e0ebd9869bdc09426f3871", - "sha256:cad9545f5801a125f162d09ec9b724b7ad9b6440151b89645241d0120e119dcc", - "sha256:cb6d161ae94fb35bb518b74bb06b7293299c15ba3bc099dccd6a5b7ae589aee3", - "sha256:d40682ac60b2a613d36d8d3a0cd14fbdf8e7e0618fbb40aa9fa7b796c9081584", - "sha256:d6128d431b8dfa888bf51c22a04d48bcb3d64431caf02b3cb943269f17fd2994", - "sha256:dbc466744a2db4b7ca05589f21ae1a35066afada2f803f92369f5877c100ef62", - "sha256:ddbef8b53cd16467fdbfa92a712eae46dd066aa19780681a2ce266e88fbc7165", - "sha256:e21cc00e4debe8f54c3ed7b9fcca540f46eee12762a9fa56feb8512fd9057161", - "sha256:eb52e826d16c09ef87132c6e360e1879c984f19a4f62d8a935345deac43f3c12", - "sha256:f0d9e7ba6a815a12c8575ba7887da4b72483e4cfc57179af10c9b937f3f9308f", - "sha256:f1e931d9a92f628858a50f5bdffdfcf839aebe388b82f9d2ccd5d22a38a789dc", - "sha256:f45808eda8b1d71308c5416ef3abe958f033fdbb356984fabbfc7887bed76b3f", - "sha256:f6d39e42a0aa888122d1beb8ec0d4ddfb6c6b45aecb5ba4013c27e2f28657765", - "sha256:fc34fdd458ff77a2a00e3c86f899911f6f269d393ca5675842a6e92eea565bae" + "sha256:019e59ef5c5256a2c7378f2fb8560fc2a9ff1d315755204295b2eab96b254d0a", + "sha256:034239843541ef7a1aee0c7b2cb7f6aafffb005ede965ae9cbd49d5ff4ff73cf", + "sha256:03b3f49b57264909aacd0741892f2aecf2f51fb053e7d8ac6767f6c700832f45", + "sha256:047a640f5c9c6ade7b1cc6680a0e28c9dd5a0825135acbd3569cc96ea00b2505", + "sha256:04ccbed567171579ec2cebb9c8a3e30801723c575601f9a990ab25bcac6b51e2", + "sha256:057e824b2aae50accc0f9a0570998adc021b372478a921506fddd6c02e60308e", + "sha256:11baebdd5fc5b475d484195e49bae2dc64b94a5208f7c89954e9e354fc609d8f", + "sha256:11c1d2aed9079c6b0c9550a7257a836b4a637feb334904610f06d70eb44c56d2", + "sha256:11d58723d44d6ed4dd677c5615b2ffb19d5c426636345567d6af82be4dff8a55", + "sha256:12720a53e61c3b99d87262294e2b375c915fea93c31fc2336898c26d7aed34cd", + "sha256:17ef5f01d25b67ca8f98120d5fa1d21efe9611604e8eb03a5147360f517dd1e2", + "sha256:18d43df3f2302d836f2a56f17e5663e398416e9dd74b205b179065e61f1a6edf", + "sha256:1a5d26fe8f32f137e784f768143728438877d69a586ddeaad898558dc971a5ae", + "sha256:1af379b33ef33757224da93e9da62e6471cf4a66d10078cf32bae8127d3d0d4a", + "sha256:1ccf825981640b8c34ae54231b7ed00271822ea1c6d8ba1090ebd4943759abf5", + "sha256:21eb4e609a154a57c520e3d5bfa0d97e49b6872ea057b7c85257b11e78068222", + "sha256:2243700cc5548cff20963f0ca92d3e5e436394375ab8a354bbea2b12911b20b0", + "sha256:255ca2b219f9e5a3a9ef3081512e1358bd4760ce77828e1028b818ff5610b87b", + "sha256:259c22485b71abacdfa8bf79720cd7bcf4b9d128b30ea554f01ae71fdbfdaa23", + "sha256:25f0e6b78220aba09815cd1f3a32b9c7cb3e02cb846d1cfc526b6595f6046618", + "sha256:273bc3959bcbff3f48606b28229b4721716598d76b5aaea2b4a9d0ab454ec062", + "sha256:292fe3fc5ad4a75bc8df0dfaee7d0babe8b1f4ceb596437213821f761b4589f9", + "sha256:2ca57a5be0389f2a65e6d3bb2962a971688cbdd30b4c0bd188c99e39c234f414", + "sha256:2d163a18819277e49911f7461567bda923461c50b19d169a062536fffe7cd9d2", + "sha256:2d81f1ddae3858b8299d1da72dd7d19dd36aab654c19671aa8a7e7fb02f6638a", + "sha256:2f957ce63d13c28730f7fd6b72333814221c84ca2421298f66e5143f81c9f91f", + "sha256:330f9e188d0d89080cde66dc7470f57d1926ff2fb5576227f14d5be7ab30b9fa", + "sha256:34c850ce7976d19ebe7b9d4b9bb8c9dfc7aac336c0958e2651b88cbd46682123", + "sha256:35b5ab8c28978fbbb86ea54958cd89f5176ce747c1fb3d87356cf698048a7790", + "sha256:3669cf8ee3520c2f13b2e0351c41fea919852b220988d2049249db10046a7afb", + "sha256:381469297409c5adf9a0e884c5eb5186ed33137badcbbb0560b86e910a2f1e76", + "sha256:3d0a409d3b28607cc427aa5c30a6f1e4452cc44e311f843e05edb28ab5e36da0", + "sha256:44e58a0554b21fc662f2712814a746635ed668d0fbc98b7cb9d74cb798d202e6", + "sha256:458dea649f2f02a0b244ae6aef8dc29325a2810aa26b07af8374dc2a9faf57e3", + "sha256:48e466162a24daf86f6b5ca72444d2bf39a5e58da5f96370078be67c67adc978", + "sha256:49d238cf4b69652257db66d0c623cd3e09b5d2e9576b56bc067a396133a00d4a", + "sha256:4ca1ed0bb2d850aa8471387882247c68f1e62a4af0ce9c8a1dbe0d2bf69e41fb", + "sha256:52533489f28d62eb1258a965f2aba28a82aa747202c8fa5a1c7a43b5db0e85c1", + "sha256:548d6482dc8aadbe7e79d1b5806585c8120bafa1ef841167bc9090522b610fa6", + "sha256:5619f3f5a4db5dbb572b095ea3cb5cc035335159d9da950830c9c4db2fbb6995", + "sha256:57459b68e5cd85b0be8184382cefd91959cafe79ae019e6b1ae6e2ba8a12cda7", + "sha256:5a34d2395073ef862b4032343cf0c32a712f3ab49d7ec4f42c9661e0294d106f", + "sha256:61706a6b6c24bdece85ff177fec393545a3191eeda35b07aaa1458a027ad1304", + "sha256:724c292bb26365659fc434e9567b3f1adbdb5e8d640c936ed901f49e03e5d32e", + "sha256:73461eed88a88c866656e08f89299720a38cb4e9d34ae6bf5df6f71102570f2e", + "sha256:76705c9325d72a81155bb6ab48d4312e0032bf045fb0754889133200f7a0d849", + "sha256:76c1c8efb3ca3a1818b837aea423ff8a07bbf7aafe9f2f6582b61a0458b1a329", + "sha256:77a41c26205d2353a4c94d02be51d6cbdf63c06fbc1295ea57dad7e2d3381b71", + "sha256:79986f3b4af059777111409ee517da24a529bdbd46da578b33f25580adcff728", + "sha256:7cff25c5b315e63b07a36f0c2bab32c58eafbe57d0dce61b614ef4c76058c115", + "sha256:7f7e58effd14b641c5e4dec8c7dab02fb67a13df90329e61c869b9cc607ef752", + "sha256:820c4a08195a681252f46926de10e29b6bbf3e17b30037bd4250d72dd3ddaab8", + "sha256:87e34f31ca8f168c56d6fbf99692cc8d3b445abb5bfd08c229ae992d7547a92a", + "sha256:8f03d3f0d01cb5a018debeb412441996a517b11c5c17ab2001aa0597c6d6882c", + "sha256:90f26dc6d5f241ba358bef79be9ce06de58d477ca8485e3291675436d3827cf8", + "sha256:955215ed0604dac5b01907424dfa28b40f2b2292d6493445dd34d0dfa72586a8", + "sha256:985bbb1316192b98f32e25e7b9958088431d853ac63aca1d2c236f40afb17c83", + "sha256:a382372898a07479bd34bda781008e4a954ed8750f17891e794521c3e21c2e1c", + "sha256:a882ac0a351288dd18ecae3326b8a49d10c61a68b01419f3a0b9a306190baf69", + "sha256:aa8d6cdc8b8aa19ceb319aaa2b660cdaccc533ec477eeb1309e2a291eaacc43a", + "sha256:abc719161780932c4e11aaebb203be3d6acc6b38d2f26c0f523b5b59d2fc1996", + "sha256:abf34e43c531bbb510ae7e8f5b2b1f2a8ab93219510e2b287a944432fad135f3", + "sha256:ade6d25bb29c4555d718ac6d1443a7386595528c33d6b133b258f65f963bb0f6", + "sha256:afea96f64efa98df4da6958bae37f1cbea7932c35878b185e5982821bc883369", + "sha256:b1579413ae492b05de5a6174574f8c44c2b9b122a42015c5292afa4be2507f28", + "sha256:b3451108ab861040754fa5208bca4a5496c65875710f76789a9ad27c801a0075", + "sha256:b9af3757495c1ee3b5c4e945c1df7be95562277c6e5bccc20a39aec50f826cd0", + "sha256:bc16ac425cc927d0a57d242589f87ee093884ea4804c05a13834d07c20db203c", + "sha256:c2910967e6ab16bf6fbeb1f771c89a7050947221ae12a5b0b60f3bca2ee19bca", + "sha256:c2b92812bd214018e50b6380ea3ac0c8bb01ac07fcc14c5f86a5bb25e74026e9", + "sha256:c2f20ce161ebdb0091a10c9ca0372e023ce24980d0e1f810f519da6f79c60800", + "sha256:c56d748ea50215abef7030c72b60dd723ed5b5c7e65e7bc2504e77843631c1a6", + "sha256:c7c133e93b405eb0d36fa430c94185bdd13c36204a8635470cccc200723c13bb", + "sha256:c9c6c9b2c2f80747a98f34ef491c4d7b1a8d4853937bb1492774992a120f475d", + "sha256:cbc8df5c6a88ba5ae385d8930da02201165408dde8d8322072e3e5ddd4f68e22", + "sha256:cff084c6933680d1f8b2f3b4ff5bbb88538a4aac00d199ac13f49d0698727ecb", + "sha256:d2045d6d9439a0078f2a34b57c7b18c4a6aef0bee37f22e4ec9f32456c852c71", + "sha256:d20a0ddb3e989e8807d83225a27e5c2eb2260eaa851532086e9e0fa0d5287d83", + "sha256:d457aed310f2670f59cc5b57dcfced452aeeed77f9da2b9763616bd57e4dbaae", + "sha256:d89528b4943d27029a2818f847c10c2cecc79fa9590f3cb1860459a5be7933eb", + "sha256:db0b2af416ba735c6304c47f75d348f498b92952f5e3e8bff449336d2728795d", + "sha256:deee9ca4727f53464daf089536e68b13e6104e84a37820a88b0a057b97bba2d2", + "sha256:df27ffddff4190667d40de7beba4a950b5ce78fe28a7dcc41d6f8a700a80a3c0", + "sha256:e0c95ddd4f6e9fca4e9e3afaa4f9df8552f0ba5d1004e89ef0a68e1f1f9807c7", + "sha256:e1c1be77bc5fb77d923850f82e55a928f8638f64a61f00ff18a67c7404faf008", + "sha256:e1ffa1c924e8c72778b9ccd386a7067cddf626884fd8277f503c48bb5f51c762", + "sha256:e2400a94f7dd9cb20cd012951a0cbf8249e3d554c63a9c0cdfd5cbb6c01d2dec", + "sha256:e61f091c3ba0c3578411ef505992d356a812fb200643eab27f4f70eed34a29ef", + "sha256:e8a701123029cc240cea61dd2d16ad57cab4691804143ce80ecd9286b464d180", + "sha256:eadbefd5e92ef8a345f0525b5cfd01cf4e4cc651a2cffb8f23c0dd184975d787", + "sha256:f32260e556a983bc5c7ed588d04c942c9a8f9c2e99213fec11a031e316874c7e", + "sha256:f8115e303280ba09f3898194791a153862cbf9eef722ad8f7f741987ee2a97c7", + "sha256:fedbdc753827cf014c01dbbee9c3be17e5a208dcd1bf8641ce2cd29580d1f0d4" ], "markers": "python_version >= '3.6'", - "version": "==25.1.0" + "version": "==25.1.1" }, "qtconsole": { "hashes": [ @@ -1497,11 +1587,11 @@ }, "referencing": { "hashes": [ - "sha256:47237742e990457f7512c7d27486394a9aadaf876cbfaa4be65b27b4f4d47c6b", - "sha256:c257b08a399b6c2f5a3510a50d28ab5dbc7bbde049bcaf954d43c446f83ab548" + "sha256:449b6669b6121a9e96a7f9e410b245d471e8d48964c67113ce9afe50c8dd7bdf", + "sha256:794ad8003c65938edcdbc027f1933215e0d0ccc0291e3ce20a4d87432b59efc0" ], "markers": "python_version >= '3.8'", - "version": "==0.30.0" + "version": "==0.30.2" }, "requests": { "hashes": [ @@ -1692,11 +1782,11 @@ }, "setuptools": { "hashes": [ - "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f", - "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235" + "sha256:d59c97e7b774979a5ccb96388efc9eb65518004537e85d52e81eaee89ab6dd91", + "sha256:e13e1b0bc760e9b0127eda042845999b2f913e12437046e663b833aa96d89715" ], - "markers": "python_version >= '3.7'", - "version": "==68.0.0" + "markers": "python_version >= '3.8'", + "version": "==68.1.0" }, "shapely": { "hashes": [ @@ -1773,6 +1863,53 @@ "markers": "python_version >= '3.7'", "version": "==2.4.1" }, + "sqlalchemy": { + "hashes": [ + "sha256:1506e988ebeaaf316f183da601f24eedd7452e163010ea63dbe52dc91c7fc70e", + "sha256:1a58052b5a93425f656675673ef1f7e005a3b72e3f2c91b8acca1b27ccadf5f4", + "sha256:1b74eeafaa11372627ce94e4dc88a6751b2b4d263015b3523e2b1e57291102f0", + "sha256:1be86ccea0c965a1e8cd6ccf6884b924c319fcc85765f16c69f1ae7148eba64b", + "sha256:1d35d49a972649b5080557c603110620a86aa11db350d7a7cb0f0a3f611948a0", + "sha256:243d0fb261f80a26774829bc2cee71df3222587ac789b7eaf6555c5b15651eed", + "sha256:26a3399eaf65e9ab2690c07bd5cf898b639e76903e0abad096cd609233ce5208", + "sha256:27d554ef5d12501898d88d255c54eef8414576f34672e02fe96d75908993cf53", + "sha256:3364b7066b3c7f4437dd345d47271f1251e0cfb0aba67e785343cdbdb0fff08c", + "sha256:3423dc2a3b94125094897118b52bdf4d37daf142cbcf26d48af284b763ab90e9", + "sha256:3c6aceebbc47db04f2d779db03afeaa2c73ea3f8dcd3987eb9efdb987ffa09a3", + "sha256:3ce5e81b800a8afc870bb8e0a275d81957e16f8c4b62415a7b386f29a0cb9763", + "sha256:411e7f140200c02c4b953b3dbd08351c9f9818d2bd591b56d0fa0716bd014f1e", + "sha256:4cde2e1096cbb3e62002efdb7050113aa5f01718035ba9f29f9d89c3758e7e4e", + "sha256:5768c268df78bacbde166b48be788b83dddaa2a5974b8810af422ddfe68a9bc8", + "sha256:599ccd23a7146e126be1c7632d1d47847fa9f333104d03325c4e15440fc7d927", + "sha256:5ed61e3463021763b853628aef8bc5d469fe12d95f82c74ef605049d810f3267", + "sha256:63a368231c53c93e2b67d0c5556a9836fdcd383f7e3026a39602aad775b14acf", + "sha256:63e73da7fb030ae0a46a9ffbeef7e892f5def4baf8064786d040d45c1d6d1dc5", + "sha256:6eb6d77c31e1bf4268b4d61b549c341cbff9842f8e115ba6904249c20cb78a61", + "sha256:6f8a934f9dfdf762c844e5164046a9cea25fabbc9ec865c023fe7f300f11ca4a", + "sha256:6fe7d61dc71119e21ddb0094ee994418c12f68c61b3d263ebaae50ea8399c4d4", + "sha256:759b51346aa388c2e606ee206c0bc6f15a5299f6174d1e10cadbe4530d3c7a98", + "sha256:76fdfc0f6f5341987474ff48e7a66c3cd2b8a71ddda01fa82fedb180b961630a", + "sha256:77d37c1b4e64c926fa3de23e8244b964aab92963d0f74d98cbc0783a9e04f501", + "sha256:79543f945be7a5ada9943d555cf9b1531cfea49241809dd1183701f94a748624", + "sha256:79fde625a0a55220d3624e64101ed68a059c1c1f126c74f08a42097a72ff66a9", + "sha256:7d3f175410a6db0ad96b10bfbb0a5530ecd4fcf1e2b5d83d968dd64791f810ed", + "sha256:8dd77fd6648b677d7742d2c3cc105a66e2681cc5e5fb247b88c7a7b78351cf74", + "sha256:a3f0dd6d15b6dc8b28a838a5c48ced7455c3e1fb47b89da9c79cc2090b072a50", + "sha256:bcb04441f370cbe6e37c2b8d79e4af9e4789f626c595899d94abebe8b38f9a4d", + "sha256:c3d99ba99007dab8233f635c32b5cd24fb1df8d64e17bc7df136cedbea427897", + "sha256:ca8a5ff2aa7f3ade6c498aaafce25b1eaeabe4e42b73e25519183e4566a16fc6", + "sha256:cb0d3e94c2a84215532d9bcf10229476ffd3b08f481c53754113b794afb62d14", + "sha256:d1b09ba72e4e6d341bb5bdd3564f1cea6095d4c3632e45dc69375a1dbe4e26ec", + "sha256:d32b5ffef6c5bcb452723a496bad2d4c52b346240c59b3e6dba279f6dcc06c14", + "sha256:d3793dcf5bc4d74ae1e9db15121250c2da476e1af8e45a1d9a52b1513a393459", + "sha256:dd81466bdbc82b060c3c110b2937ab65ace41dfa7b18681fdfad2f37f27acdd7", + "sha256:e4e571af672e1bb710b3cc1a9794b55bce1eae5aed41a608c0401885e3491179", + "sha256:ea8186be85da6587456c9ddc7bf480ebad1a0e6dcbad3967c4821233a4d4df57", + "sha256:eefebcc5c555803065128401a1e224a64607259b5eb907021bf9b175f315d2a6" + ], + "index": "pypi", + "version": "==2.0.20" + }, "stack-data": { "hashes": [ "sha256:32d2dd0376772d01b6cb9fc996f3c8b57a357089dec328ed4b6553d037eaf815", @@ -1806,20 +1943,20 @@ }, "tornado": { "hashes": [ - "sha256:05615096845cf50a895026f749195bf0b10b8909f9be672f50b0fe69cba368e4", - "sha256:0c325e66c8123c606eea33084976c832aa4e766b7dff8aedd7587ea44a604cdf", - "sha256:29e71c847a35f6e10ca3b5c2990a52ce38b233019d8e858b755ea6ce4dcdd19d", - "sha256:4b927c4f19b71e627b13f3db2324e4ae660527143f9e1f2e2fb404f3a187e2ba", - "sha256:5b17b1cf5f8354efa3d37c6e28fdfd9c1c1e5122f2cb56dac121ac61baa47cbe", - "sha256:6a0848f1aea0d196a7c4f6772197cbe2abc4266f836b0aac76947872cd29b411", - "sha256:7efcbcc30b7c654eb6a8c9c9da787a851c18f8ccd4a5a3a95b05c7accfa068d2", - "sha256:834ae7540ad3a83199a8da8f9f2d383e3c3d5130a328889e4cc991acc81e87a0", - "sha256:b46a6ab20f5c7c1cb949c72c1994a4585d2eaa0be4853f50a03b5031e964fc7c", - "sha256:c2de14066c4a38b4ecbbcd55c5cc4b5340eb04f1c5e81da7451ef555859c833f", - "sha256:c367ab6c0393d71171123ca5515c61ff62fe09024fa6bf299cd1339dc9456829" + "sha256:1bd19ca6c16882e4d37368e0152f99c099bad93e0950ce55e71daed74045908f", + "sha256:22d3c2fa10b5793da13c807e6fc38ff49a4f6e1e3868b0a6f4164768bb8e20f5", + "sha256:502fba735c84450974fec147340016ad928d29f1e91f49be168c0a4c18181e1d", + "sha256:65ceca9500383fbdf33a98c0087cb975b2ef3bfb874cb35b8de8740cf7f41bd3", + "sha256:71a8db65160a3c55d61839b7302a9a400074c9c753040455494e2af74e2501f2", + "sha256:7ac51f42808cca9b3613f51ffe2a965c8525cb1b00b7b2d56828b8045354f76a", + "sha256:7d01abc57ea0dbb51ddfed477dfe22719d376119844e33c661d873bf9c0e4a16", + "sha256:805d507b1f588320c26f7f097108eb4023bbaa984d63176d1652e184ba24270a", + "sha256:9dc4444c0defcd3929d5c1eb5706cbe1b116e762ff3e0deca8b715d14bf6ec17", + "sha256:ceb917a50cd35882b57600709dd5421a418c29ddc852da8bcdab1f0db33406b0", + "sha256:e7d8db41c0181c80d76c982aacc442c0783a2c54d6400fe028954201a2e032fe" ], "markers": "python_version >= '3.8'", - "version": "==6.3.2" + "version": "==6.3.3" }, "traitlets": { "hashes": [ @@ -1829,6 +1966,14 @@ "markers": "python_version >= '3.7'", "version": "==5.9.0" }, + "typing-extensions": { + "hashes": [ + "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36", + "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2" + ], + "markers": "python_version >= '3.7'", + "version": "==4.7.1" + }, "tzdata": { "hashes": [ "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a", diff --git a/README.md b/README.md index bb2bf54f..4aa9406b 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,36 @@ # vacant-lots-proj + Gun crime x vacant lots mapping tool layer # Setup Instructions + 1. Install [pyenv](https://github.com/pyenv/pyenv) (or [pyenv-win](https://github.com/pyenv-win/pyenv-win) for Windows) for Python version management 2. Install the latest Python 3.11: `pyenv install 3.11.4` 3. Install [pipenv](https://github.com/pypa/pipenv) for environment management 4. Install project requirements: `pipenv install` +# Database + +1. Make sure postgres and postgis are installed + TODO: Add instructions here +2. Create a new databse + `createdb vacantlotdb` +3. Connect to the db + `psql vacantlotdb` +4. Enable postgis + `CREATE EXTENSION postgis;` +5. Set your database connection string to an environment variable `VACANT_LOTS_DB` + # Usage Instructions + 1. Activate the pipenv shell: `pipenv shell` 2. Run the main script: `python process_data.py [output_filename]` # Contribution Instructions + 1. Create a fork of this repository and work from branches within your fork. When those changes are ready for review, create a pull request from fork:branch to upstream:main -2. Before committing changes, format your code to maintain a consistent codebase: - ``` - pipenv shell - black . - ``` +2. Before committing changes, format your code to maintain a consistent codebase: + ``` + pipenv shell + black . + ``` diff --git a/2023_02_02_vacant_lots_layer.ipynb b/archived/2023_02_02_vacant_lots_layer.ipynb similarity index 100% rename from 2023_02_02_vacant_lots_layer.ipynb rename to archived/2023_02_02_vacant_lots_layer.ipynb diff --git a/constants.py b/archived/constants.py similarity index 100% rename from constants.py rename to archived/constants.py diff --git a/data_utils.py b/archived/data_utils.py similarity index 100% rename from data_utils.py rename to archived/data_utils.py diff --git a/process_data.py b/archived/process_data.py similarity index 100% rename from process_data.py rename to archived/process_data.py diff --git a/notes.md b/notes.md new file mode 100644 index 00000000..aa015d88 --- /dev/null +++ b/notes.md @@ -0,0 +1 @@ +L&I complaints- merge by intersect instead of on address diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/classes/__init__.py b/src/classes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/script.py b/src/classes/featurelayer.py similarity index 60% rename from script.py rename to src/classes/featurelayer.py index 18fc9c53..5d3a9dfd 100644 --- a/script.py +++ b/src/classes/featurelayer.py @@ -1,12 +1,10 @@ import requests import geopandas as gpd import pandas as pd -import datetime from esridump.dumper import EsriDumper - - -# Set the CRS to use for all layers -use_crs = "EPSG:2272" +import traceback +from config.config import USE_CRS, FORCE_RELOAD +from config.psql import conn class FeatureLayer: @@ -15,7 +13,14 @@ class FeatureLayer: """ def __init__( - self, name, esri_rest_urls=None, carto_sql_queries=None, gdf=None, crs=use_crs + self, + name, + esri_rest_urls=None, + carto_sql_queries=None, + gdf=None, + crs=USE_CRS, + force_reload=FORCE_RELOAD, + from_xy=False, ): self.name = name self.esri_rest_urls = ( @@ -28,6 +33,8 @@ def __init__( ) self.gdf = gdf self.crs = crs + self.psql_table = name.lower().replace(" ", "_") + self.input_crs = "EPSG:4326" if not from_xy else USE_CRS inputs = [self.esri_rest_urls, self.carto_sql_queries, self.gdf] non_none_inputs = [i for i in inputs if i is not None] @@ -44,10 +51,29 @@ def __init__( elif self.gdf is not None: self.type = "gdf" - self.load_data() + if force_reload: + self.load_data() + else: + psql_exists = self.check_psql() + if not psql_exists: + self.load_data() + + def check_psql(self): + try: + psql_table = gpd.read_postgis( + f"SELECT * FROM {self.psql_table}", conn, geom_col="geometry" + ) + if len(psql_table) == 0: + return False + else: + print(f"Loading data for {self.name} from psql...") + self.gdf = psql_table + return True + except: + return False def load_data(self): - print(f"Loading data for {self.name}...") + print(f"Loading data for {self.name} from {self.type}...") if self.type == "gdf": self.gdf = gdf else: @@ -65,11 +91,12 @@ def load_data(self): "type": "FeatureCollection", "features": features, } - gdfs.append( - gpd.GeoDataFrame.from_features( - geojson_features, crs=self.crs - ) + + this_gdf = gpd.GeoDataFrame.from_features( + geojson_features, crs=self.input_crs ) + this_gdf = this_gdf.to_crs(USE_CRS) + gdfs.append(this_gdf) self.gdf = pd.concat(gdfs, ignore_index=True) @@ -87,15 +114,25 @@ def load_data(self): data = response.json()["rows"] df = pd.DataFrame(data) + gdf = gpd.GeoDataFrame( - df, geometry=gpd.points_from_xy(df.x, df.y), crs=self.crs + df, + geometry=gpd.points_from_xy(df.x, df.y), + crs=self.input_crs, ) + gdf = gdf.to_crs(USE_CRS) gdfs.append(gdf) self.gdf = pd.concat(gdfs, ignore_index=True) + # save self.gdf to psql + self.gdf.to_postgis( + name=self.psql_table, con=conn, if_exists="replace", chunksize=1000 + ) + except Exception as e: print(f"Error loading data for {self.name}: {e}") + traceback.print_exc() self.gdf = None def spatial_join(self, other_layer, how="left", predicate="intersects"): @@ -104,61 +141,18 @@ def spatial_join(self, other_layer, how="left", predicate="intersects"): They also can create duplicates, so we drop duplicates after the join. Note: We may want to revisit the duplicates. """ - self.gdf = gpd.sjoin(self.gdf, other_layer.gdf, how=how, predicate=predicate) - self.gdf.drop(columns=["index_right"], inplace=True) - self.gdf.drop_duplicates(inplace=True) - - -""" -Load Vacant Properties Datasets -""" - -vacant_props_layers_to_load = [ - "https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/Vacant_Indicators_Land/FeatureServer/0/", - "https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/Vacant_Indicators_Bldg/FeatureServer/0/", -] - -vacant_properties = FeatureLayer( - name="Vacant Properties", esri_rest_urls=vacant_props_layers_to_load -) - - -""" -Load City Owned Properties -""" -city_owned_properties = FeatureLayer( - name="City Owned Properties", - esri_rest_urls="https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/LAMAAssets/FeatureServer/0/", -) - -# Discover which vacant properties are city owned using a spatial join -vacant_properties.spatial_join(city_owned_properties) - -# There are some matches which aren't valid because of imprecise parcel boundaries. Use a match on OPA_ID and OPABRT to remove these. -vacant_properties.gdf = vacant_properties.gdf.loc[ - ~( - (vacant_properties.gdf["OPABRT"].notnull()) - & (vacant_properties.gdf["OPA_ID"] != vacant_properties.gdf["OPABRT"]) - ) -] - -# Note: This removes some entries from the dataset, need to revisit this - - -""" -Load PHS Data -""" - -phs_layers_to_load = [ - "https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/PHS_CommunityLandcare/FeatureServer/0/", - "https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/PHS_PhilaLandCare_Maintenance/FeatureServer/0/", -] - -phs_properties = FeatureLayer(name="PHS Properties", esri_rest_urls=phs_layers_to_load) + # If other_layer.gdf isn't a geodataframe, try to make it one + if not isinstance(other_layer.gdf, gpd.GeoDataFrame): + try: + other_layer.gdf = gpd.GeoDataFrame(other_layer.gdf, geometry="geometry") -phs_properties.gdf["COMM_PARTN"] = "PHS" -phs_properties.gdf = phs_properties.gdf[["COMM_PARTN", "geometry"]] + except: + print(other_layer.gdf) + raise ValueError( + "other_layer.gdf must be a GeoDataFrame or a DataFrame with x and y columns." + ) -vacant_properties.spatial_join(phs_properties) -vacant_properties.gdf["COMM_PARTN"].fillna("None", inplace=True) + self.gdf = gpd.sjoin(self.gdf, other_layer.gdf, how=how, predicate=predicate) + self.gdf.drop(columns=["index_right"], inplace=True) + self.gdf.drop_duplicates(inplace=True) diff --git a/src/config/__init__.py b/src/config/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/config/config.py b/src/config/config.py new file mode 100644 index 00000000..5ee847d2 --- /dev/null +++ b/src/config/config.py @@ -0,0 +1,2 @@ +FORCE_RELOAD = True +USE_CRS = "EPSG:2272" diff --git a/src/config/psql.py b/src/config/psql.py new file mode 100644 index 00000000..5553391d --- /dev/null +++ b/src/config/psql.py @@ -0,0 +1,6 @@ +import os +from sqlalchemy import create_engine + +# Connect to psql +engine = create_engine(os.environ["VACANT_LOTS_DB"]) +conn = engine.connect() diff --git a/src/constants/__init__.py b/src/constants/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/constants/services.py b/src/constants/services.py new file mode 100644 index 00000000..c5ab1d6a --- /dev/null +++ b/src/constants/services.py @@ -0,0 +1,24 @@ +import datetime + +VACANT_PROPS_LAYERS_TO_LOAD = [ + "https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/Vacant_Indicators_Land/FeatureServer/0/", + "https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/Vacant_Indicators_Bldg/FeatureServer/0/", +] + +CITY_OWNED_PROPERTIES_TO_LOAD = [ + "https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/LAMAAssets/FeatureServer/0/" +] + +PHS_LAYERS_TO_LOAD = [ + "https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/PHS_CommunityLandcare/FeatureServer/0/", + "https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/PHS_PhilaLandCare_Maintenance/FeatureServer/0/", +] + +one_year_ago = (datetime.datetime.now() - datetime.timedelta(days=365)).strftime( + "%Y-%m-%d" +) + +# Load data for complaints from L&I +COMPLAINTS_SQL_QUERY = f"SELECT address, service_request_id, subject, status, service_name, service_code, lat AS y, lon AS x FROM public_cases_fc WHERE requested_datetime >= '{one_year_ago}' AND lat IS NOT NULL;" + +VIOLATIONS_SQL_QUERY = f"SELECT parcel_id_num, casenumber, casecreateddate, casetype, casestatus, violationnumber, violationcodetitle, violationstatus, opa_account_num, address, opa_owner, geocode_x AS x, geocode_y AS y FROM violations WHERE violationdate >= '{one_year_ago}' AND geocode_x IS NOT NULL;" diff --git a/src/data_utils/__init__.py b/src/data_utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/data_utils/city_owned_properties.py b/src/data_utils/city_owned_properties.py new file mode 100644 index 00000000..5812446c --- /dev/null +++ b/src/data_utils/city_owned_properties.py @@ -0,0 +1,21 @@ +from classes.featurelayer import FeatureLayer +from constants.services import CITY_OWNED_PROPERTIES_TO_LOAD + + +def city_owned_properties(primary_featurelayer): + city_owned_properties = FeatureLayer( + name="City Owned Properties", + esri_rest_urls=CITY_OWNED_PROPERTIES_TO_LOAD, + ) + + # Discover which vacant properties are city owned using a spatial join + primary_featurelayer.spatial_join(city_owned_properties) + + # There are some matches which aren't valid because of imprecise parcel boundaries. Use a match on OPA_ID and OPABRT to remove these. + primary_featurelayer.gdf = primary_featurelayer.gdf.loc[ + ~( + (primary_featurelayer.gdf["OPABRT"].notnull()) + & (primary_featurelayer.gdf["OPA_ID"] != primary_featurelayer.gdf["OPABRT"]) + ) + ] + return primary_featurelayer diff --git a/src/data_utils/l_and_i.py b/src/data_utils/l_and_i.py new file mode 100644 index 00000000..16389ab3 --- /dev/null +++ b/src/data_utils/l_and_i.py @@ -0,0 +1,98 @@ +from classes.featurelayer import FeatureLayer +from constants.services import COMPLAINTS_SQL_QUERY, VIOLATIONS_SQL_QUERY +import geopandas as gpd + + +def l_and_i(primary_featurelayer): + # Calculate one year ago from today's date + l_and_i_complaints = FeatureLayer( + name="LI Complaints", carto_sql_queries=COMPLAINTS_SQL_QUERY + ) + + # filter for only Status = 'Open' + l_and_i_complaints.gdf = l_and_i_complaints.gdf[ + l_and_i_complaints.gdf["status"] == "Open" + ] + + # Group by geometry and concatenate the violationcodetitle values into a list with a semicolon separator + l_and_i_complaints.gdf = ( + l_and_i_complaints.gdf.groupby("geometry")["service_name"] + .apply(lambda x: "; ".join([val for val in x if val is not None])) + .reset_index() + ) + l_and_i_complaints.gdf = gpd.GeoDataFrame( + l_and_i_complaints.gdf, geometry="geometry" + ) + + # collapse complaints_gdf by address and concatenate the violationcodetitle values into a list with a semicolon separator + # l_and_i_complaints.gdf = l_and_i_complaints.gdf.groupby('address')['service_name'].apply(lambda x: '; '.join([val for val in x if val is not None])).reset_index() + + # rename the column to 'li_complaints' + l_and_i_complaints.gdf.rename( + columns={"service_name": "li_complaints"}, inplace=True + ) + + # Load data for violations from L&I + l_and_i_violations = FeatureLayer( + name="LI Violations", carto_sql_queries=VIOLATIONS_SQL_QUERY, from_xy=True + ) + + all_violations_count_df = ( + l_and_i_violations.gdf.groupby("opa_account_num") + .count() + .reset_index()[["opa_account_num", "violationnumber", "geometry"]] + ) + all_violations_count_df = all_violations_count_df.rename( + columns={"violationnumber": "all_violations_past_year"} + ) + # filter for only cases where the casestatus is 'IN VIOLATION' or 'UNDER INVESTIGATION' + violations_gdf = l_and_i_violations.gdf[ + (l_and_i_violations.gdf["violationstatus"] == "OPEN") + ] + + open_violations_count_df = ( + violations_gdf.groupby("opa_account_num") + .count() + .reset_index()[["opa_account_num", "violationnumber", "geometry"]] + ) + open_violations_count_df = open_violations_count_df.rename( + columns={"violationnumber": "open_violations_past_year"} + ) + # join the all_violations_count_df and open_violations_count_df dataframes on opa_account_num + violations_count_gdf = all_violations_count_df.merge( + open_violations_count_df, how="left", on="opa_account_num" + ) + + # replace NaN values with 0 + violations_count_gdf.fillna(0, inplace=True) + + # convert the all_violations_past_year and open_violations_past_year columns to integers + violations_count_gdf["all_violations_past_year"] = violations_count_gdf[ + "all_violations_past_year" + ].astype(int) + violations_count_gdf["open_violations_past_year"] = violations_count_gdf[ + "open_violations_past_year" + ].astype(int) + + # collapse violations_gdf by address and concatenate the violationcodetitle values into a list with a semicolon separator + l_and_i_violations.gdf = ( + l_and_i_violations.gdf.groupby("geometry")["violationcodetitle"] + .apply(lambda x: "; ".join([val for val in x if val is not None])) + .reset_index() + ) + l_and_i_complaints.gdf = gpd.GeoDataFrame( + l_and_i_complaints.gdf, geometry="geometry" + ) + + # rename the column to 'li_violations' + l_and_i_violations.gdf.rename( + columns={"violationcodetitle": "li_code_violations"}, inplace=True + ) + + # left join the complaints_gdf to the joined_gdf on address + primary_featurelayer.spatial_join(l_and_i_violations, predicate="contains") + + # left join the violations_gdf to the joined_gdf on opa_account_num + primary_featurelayer.spatial_join(l_and_i_complaints, predicate="contains") + + return primary_featurelayer diff --git a/src/data_utils/phs_properties.py b/src/data_utils/phs_properties.py new file mode 100644 index 00000000..58c72713 --- /dev/null +++ b/src/data_utils/phs_properties.py @@ -0,0 +1,16 @@ +from classes.featurelayer import FeatureLayer +from constants.services import PHS_LAYERS_TO_LOAD + + +def phs_properties(primary_featurelayer): + phs_properties = FeatureLayer( + name="PHS Properties", esri_rest_urls=PHS_LAYERS_TO_LOAD + ) + + phs_properties.gdf["COMM_PARTN"] = "PHS" + phs_properties.gdf = phs_properties.gdf[["COMM_PARTN", "geometry"]] + + primary_featurelayer.spatial_join(phs_properties) + primary_featurelayer.gdf["COMM_PARTN"].fillna("None", inplace=True) + + return primary_featurelayer diff --git a/src/script.py b/src/script.py new file mode 100644 index 00000000..15a43794 --- /dev/null +++ b/src/script.py @@ -0,0 +1,42 @@ +from classes.featurelayer import FeatureLayer +from config.psql import conn +from constants.services import VACANT_PROPS_LAYERS_TO_LOAD +from data_utils.city_owned_properties import city_owned_properties +from data_utils.phs_properties import phs_properties +from data_utils.l_and_i import l_and_i + + +""" +Load Vacant Properties Datasets +""" + + +vacant_properties = FeatureLayer( + name="Vacant Properties", esri_rest_urls=VACANT_PROPS_LAYERS_TO_LOAD +) + + +""" +Load City Owned Properties +""" +vacant_properties = city_owned_properties(vacant_properties) + + +""" +Load PHS Data +""" +vacant_properties = phs_properties(vacant_properties) + + +""" +Load L&I Data +""" +vacant_properties = l_and_i(vacant_properties) + +vacant_properties.gdf.to_postgis( + "vacant_properties_end", conn, if_exists="replace", index=False +) + +# Clean up + +conn.close() From d24d56a70ee2e78e29c070baf98f0647e1fc4bde Mon Sep 17 00:00:00 2001 From: Brandon Cohen Date: Sun, 27 Aug 2023 20:19:41 -0400 Subject: [PATCH 5/7] ready for PR --- .gitignore | 1 + Pipfile.lock | 316 +++++++++++++++++++--------------- README.md | 7 +- archived/constants.py | 97 ----------- archived/data_utils.py | 248 -------------------------- archived/process_data.py | 267 ---------------------------- notes.md | 1 - src/classes/featurelayer.py | 37 ++-- src/constants/services.py | 6 + src/data_utils/gun_crimes.py | 80 +++++++++ src/data_utils/l_and_i.py | 9 +- src/data_utils/rco_geoms.py | 18 ++ src/data_utils/tree_canopy.py | 30 ++++ src/script.py | 26 ++- 14 files changed, 364 insertions(+), 779 deletions(-) delete mode 100644 archived/constants.py delete mode 100644 archived/data_utils.py delete mode 100644 archived/process_data.py delete mode 100644 notes.md create mode 100644 src/data_utils/gun_crimes.py create mode 100644 src/data_utils/rco_geoms.py create mode 100644 src/data_utils/tree_canopy.py diff --git a/.gitignore b/.gitignore index e2d7d547..05e29933 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /notebooks/.ipynb_checkpoints/ .idea +src/tmp \ No newline at end of file diff --git a/Pipfile.lock b/Pipfile.lock index f614ef52..a25c38c4 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -326,11 +326,11 @@ }, "click": { "hashes": [ - "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd", - "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5" + "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", + "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de" ], "markers": "python_version >= '3.7'", - "version": "==8.1.6" + "version": "==8.1.7" }, "click-plugins": { "hashes": [ @@ -498,43 +498,43 @@ }, "fonttools": { "hashes": [ - "sha256:01cfe02416b6d416c5c8d15e30315cbcd3e97d1b50d3b34b0ce59f742ef55258", - "sha256:0a1466713e54bdbf5521f2f73eebfe727a528905ff5ec63cda40961b4b1eea95", - "sha256:0df8ef75ba5791e873c9eac2262196497525e3f07699a2576d3ab9ddf41cb619", - "sha256:10dac980f2b975ef74532e2a94bb00e97a95b4595fb7f98db493c474d5f54d0e", - "sha256:150122ed93127a26bc3670ebab7e2add1e0983d30927733aec327ebf4255b072", - "sha256:1f81ed9065b4bd3f4f3ce8e4873cd6a6b3f4e92b1eddefde35d332c6f414acc3", - "sha256:27ec3246a088555629f9f0902f7412220c67340553ca91eb540cf247aacb1983", - "sha256:2d6dc3fa91414ff4daa195c05f946e6a575bd214821e26d17ca50f74b35b0fe4", - "sha256:329341ba3d86a36e482610db56b30705384cb23bd595eac8cbb045f627778e9d", - "sha256:3fb2a69870bfe143ec20b039a1c8009e149dd7780dd89554cc8a11f79e5de86b", - "sha256:4655c480a1a4d706152ff54f20e20cf7609084016f1df3851cce67cef768f40a", - "sha256:48e82d776d2e93f88ca56567509d102266e7ab2fb707a0326f032fe657335238", - "sha256:57b68eab183fafac7cd7d464a7bfa0fcd4edf6c67837d14fb09c1c20516cf20b", - "sha256:58c1165f9b2662645de9b19a8c8bdd636b36294ccc07e1b0163856b74f10bafc", - "sha256:614b1283dca88effd20ee48160518e6de275ce9b5456a3134d5f235523fc5065", - "sha256:685a4dd6cf31593b50d6d441feb7781a4a7ef61e19551463e14ed7c527b86f9f", - "sha256:6bd7e4777bff1dcb7c4eff4786998422770f3bfbef8be401c5332895517ba3fa", - "sha256:703101eb0490fae32baf385385d47787b73d9ea55253df43b487c89ec767e0d7", - "sha256:83b98be5d291e08501bd4fc0c4e0f8e6e05b99f3924068b17c5c9972af6fff84", - "sha256:8ece1886d12bb36c48c00b2031518877f41abae317e3a55620d38e307d799b7e", - "sha256:9c456d1f23deff64ffc8b5b098718e149279abdea4d8692dba69172fb6a0d597", - "sha256:9cd2363ea7728496827658682d049ffb2e98525e2247ca64554864a8cc945568", - "sha256:a9b55d2a3b360e0c7fc5bd8badf1503ca1c11dd3a1cd20f2c26787ffa145a9c7", - "sha256:ae7df0ae9ee2f3f7676b0ff6f4ebe48ad0acaeeeaa0b6839d15dbf0709f2c5ef", - "sha256:ae881e484702efdb6cf756462622de81d4414c454edfd950b137e9a7352b3cb9", - "sha256:b8600ae7dce6ec3ddfb201abb98c9d53abbf8064d7ac0c8a0d8925e722ccf2a0", - "sha256:c36c904ce0322df01e590ba814d5d69e084e985d7e4c2869378671d79662a7d4", - "sha256:c8bf88f9e3ce347c716921804ef3a8330cb128284eb6c0b6c4b3574f3c580023", - "sha256:d40673b2e927f7cd0819c6f04489dfbeb337b4a7b10fc633c89bf4f34ecb9620", - "sha256:d54e600a2bcfa5cdaa860237765c01804a03b08404d6affcd92942fa7315ffba", - "sha256:dfe7fa7e607f7e8b58d0c32501a3a7cac148538300626d1b930082c90ae7f6bd", - "sha256:e35bed436726194c5e6e094fdfb423fb7afaa0211199f9d245e59e11118c576c", - "sha256:f0290ea7f9945174bd4dfd66e96149037441eb2008f3649094f056201d99e293", - "sha256:fae4e801b774cc62cecf4a57b1eae4097903fced00c608d9e2bc8f84cd87b54a" + "sha256:0eb79a2da5eb6457a6f8ab904838454accc7d4cccdaff1fd2bd3a0679ea33d64", + "sha256:113337c2d29665839b7d90b39f99b3cac731f72a0eda9306165a305c7c31d341", + "sha256:12a7c247d1b946829bfa2f331107a629ea77dc5391dfd34fdcd78efa61f354ca", + "sha256:179737095eb98332a2744e8f12037b2977f22948cf23ff96656928923ddf560a", + "sha256:19b7db825c8adee96fac0692e6e1ecd858cae9affb3b4812cdb9d934a898b29e", + "sha256:37983b6bdab42c501202500a2be3a572f50d4efe3237e0686ee9d5f794d76b35", + "sha256:3a35981d90feebeaef05e46e33e6b9e5b5e618504672ca9cd0ff96b171e4bfff", + "sha256:46a0ec8adbc6ff13494eb0c9c2e643b6f009ce7320cf640de106fb614e4d4360", + "sha256:4aa79366e442dbca6e2c8595645a3a605d9eeabdb7a094d745ed6106816bef5d", + "sha256:515607ec756d7865f23070682622c49d922901943697871fc292277cf1e71967", + "sha256:53eb5091ddc8b1199330bb7b4a8a2e7995ad5d43376cadce84523d8223ef3136", + "sha256:5d18fc642fd0ac29236ff88ecfccff229ec0386090a839dd3f1162e9a7944a40", + "sha256:5fb289b7a815638a7613d46bcf324c9106804725b2bb8ad913c12b6958ffc4ec", + "sha256:62f481ac772fd68901573956231aea3e4b1ad87b9b1089a61613a91e2b50bb9b", + "sha256:689508b918332fb40ce117131633647731d098b1b10d092234aa959b4251add5", + "sha256:68a02bbe020dc22ee0540e040117535f06df9358106d3775e8817d826047f3fd", + "sha256:6ed2662a3d9c832afa36405f8748c250be94ae5dfc5283d668308391f2102861", + "sha256:7286aed4ea271df9eab8d7a9b29e507094b51397812f7ce051ecd77915a6e26b", + "sha256:7cc7d685b8eeca7ae69dc6416833fbfea61660684b7089bca666067cb2937dcf", + "sha256:8708b98c278012ad267ee8a7433baeb809948855e81922878118464b274c909d", + "sha256:9398f244e28e0596e2ee6024f808b06060109e33ed38dcc9bded452fd9bbb853", + "sha256:9e36344e48af3e3bde867a1ca54f97c308735dd8697005c2d24a86054a114a71", + "sha256:a398bdadb055f8de69f62b0fc70625f7cbdab436bbb31eef5816e28cab083ee8", + "sha256:acb47f6f8680de24c1ab65ebde39dd035768e2a9b571a07c7b8da95f6c8815fd", + "sha256:be24fcb80493b2c94eae21df70017351851652a37de514de553435b256b2f249", + "sha256:c391cd5af88aacaf41dd7cfb96eeedfad297b5899a39e12f4c2c3706d0a3329d", + "sha256:c95b0724a6deea2c8c5d3222191783ced0a2f09bd6d33f93e563f6f1a4b3b3a4", + "sha256:c9b1ce7a45978b821a06d375b83763b27a3a5e8a2e4570b3065abad240a18760", + "sha256:db372213d39fa33af667c2aa586a0c1235e88e9c850f5dd5c8e1f17515861868", + "sha256:db55cbaea02a20b49fefbd8e9d62bd481aaabe1f2301dabc575acc6b358874fa", + "sha256:ed1a13a27f59d1fc1920394a7f596792e9d546c9ca5a044419dca70c37815d7c", + "sha256:f2b82f46917d8722e6b5eafeefb4fb585d23babd15d8246c664cd88a5bddd19c", + "sha256:f2f806990160d1ce42d287aa419df3ffc42dfefe60d473695fb048355fe0c6a0", + "sha256:f720fa82a11c0f9042376fd509b5ed88dab7e3cd602eee63a1af08883b37342b" ], "markers": "python_version >= '3.8'", - "version": "==4.42.0" + "version": "==4.42.1" }, "fqdn": { "hashes": [ @@ -710,11 +710,11 @@ }, "jupyter-server": { "hashes": [ - "sha256:59838bf20759354e2b4222fed702948a934cb21a503e21cd5b03706a456391d9", - "sha256:76b4ae0b568c331acc9aa904fc4a1194ef9bdefa69f232deb3f3cae802528c05" + "sha256:98a375347b580e837e7016007c24680a4261ed8ad7cd35196ac087d229f48e5a", + "sha256:d64fb4e593907290e5df916e3c9399c15ab2cd7bdb71cbcd1d36452dbfb30523" ], "markers": "python_version >= '3.8'", - "version": "==2.7.1" + "version": "==2.7.2" }, "jupyter-server-terminals": { "hashes": [ @@ -758,77 +758,113 @@ }, "kiwisolver": { "hashes": [ - "sha256:02f79693ec433cb4b5f51694e8477ae83b3205768a6fb48ffba60549080e295b", - "sha256:03baab2d6b4a54ddbb43bba1a3a2d1627e82d205c5cf8f4c924dc49284b87166", - "sha256:1041feb4cda8708ce73bb4dcb9ce1ccf49d553bf87c3954bdfa46f0c3f77252c", - "sha256:10ee06759482c78bdb864f4109886dff7b8a56529bc1609d4f1112b93fe6423c", - "sha256:1d1573129aa0fd901076e2bfb4275a35f5b7aa60fbfb984499d661ec950320b0", - "sha256:283dffbf061a4ec60391d51e6155e372a1f7a4f5b15d59c8505339454f8989e4", - "sha256:28bc5b299f48150b5f822ce68624e445040595a4ac3d59251703779836eceff9", - "sha256:2a66fdfb34e05b705620dd567f5a03f239a088d5a3f321e7b6ac3239d22aa286", - "sha256:2e307eb9bd99801f82789b44bb45e9f541961831c7311521b13a6c85afc09767", - "sha256:2e407cb4bd5a13984a6c2c0fe1845e4e41e96f183e5e5cd4d77a857d9693494c", - "sha256:2f5e60fabb7343a836360c4f0919b8cd0d6dbf08ad2ca6b9cf90bf0c76a3c4f6", - "sha256:36dafec3d6d6088d34e2de6b85f9d8e2324eb734162fba59d2ba9ed7a2043d5b", - "sha256:3fe20f63c9ecee44560d0e7f116b3a747a5d7203376abeea292ab3152334d004", - "sha256:41dae968a94b1ef1897cb322b39360a0812661dba7c682aa45098eb8e193dbdf", - "sha256:4bd472dbe5e136f96a4b18f295d159d7f26fd399136f5b17b08c4e5f498cd494", - "sha256:4ea39b0ccc4f5d803e3337dd46bcce60b702be4d86fd0b3d7531ef10fd99a1ac", - "sha256:5853eb494c71e267912275e5586fe281444eb5e722de4e131cddf9d442615626", - "sha256:5bce61af018b0cb2055e0e72e7d65290d822d3feee430b7b8203d8a855e78766", - "sha256:6295ecd49304dcf3bfbfa45d9a081c96509e95f4b9d0eb7ee4ec0530c4a96514", - "sha256:62ac9cc684da4cf1778d07a89bf5f81b35834cb96ca523d3a7fb32509380cbf6", - "sha256:70e7c2e7b750585569564e2e5ca9845acfaa5da56ac46df68414f29fea97be9f", - "sha256:7577c1987baa3adc4b3c62c33bd1118c3ef5c8ddef36f0f2c950ae0b199e100d", - "sha256:75facbe9606748f43428fc91a43edb46c7ff68889b91fa31f53b58894503a191", - "sha256:787518a6789009c159453da4d6b683f468ef7a65bbde796bcea803ccf191058d", - "sha256:78d6601aed50c74e0ef02f4204da1816147a6d3fbdc8b3872d263338a9052c51", - "sha256:7c43e1e1206cd421cd92e6b3280d4385d41d7166b3ed577ac20444b6995a445f", - "sha256:81e38381b782cc7e1e46c4e14cd997ee6040768101aefc8fa3c24a4cc58e98f8", - "sha256:841293b17ad704d70c578f1f0013c890e219952169ce8a24ebc063eecf775454", - "sha256:872b8ca05c40d309ed13eb2e582cab0c5a05e81e987ab9c521bf05ad1d5cf5cb", - "sha256:877272cf6b4b7e94c9614f9b10140e198d2186363728ed0f701c6eee1baec1da", - "sha256:8c808594c88a025d4e322d5bb549282c93c8e1ba71b790f539567932722d7bd8", - "sha256:8ed58b8acf29798b036d347791141767ccf65eee7f26bde03a71c944449e53de", - "sha256:91672bacaa030f92fc2f43b620d7b337fd9a5af28b0d6ed3f77afc43c4a64b5a", - "sha256:968f44fdbf6dd757d12920d63b566eeb4d5b395fd2d00d29d7ef00a00582aac9", - "sha256:9f85003f5dfa867e86d53fac6f7e6f30c045673fa27b603c397753bebadc3008", - "sha256:a553dadda40fef6bfa1456dc4be49b113aa92c2a9a9e8711e955618cd69622e3", - "sha256:a68b62a02953b9841730db7797422f983935aeefceb1679f0fc85cbfbd311c32", - "sha256:abbe9fa13da955feb8202e215c4018f4bb57469b1b78c7a4c5c7b93001699938", - "sha256:ad881edc7ccb9d65b0224f4e4d05a1e85cf62d73aab798943df6d48ab0cd79a1", - "sha256:b1792d939ec70abe76f5054d3f36ed5656021dcad1322d1cc996d4e54165cef9", - "sha256:b428ef021242344340460fa4c9185d0b1f66fbdbfecc6c63eff4b7c29fad429d", - "sha256:b533558eae785e33e8c148a8d9921692a9fe5aa516efbdff8606e7d87b9d5824", - "sha256:ba59c92039ec0a66103b1d5fe588fa546373587a7d68f5c96f743c3396afc04b", - "sha256:bc8d3bd6c72b2dd9decf16ce70e20abcb3274ba01b4e1c96031e0c4067d1e7cd", - "sha256:bc9db8a3efb3e403e4ecc6cd9489ea2bac94244f80c78e27c31dcc00d2790ac2", - "sha256:bf7d9fce9bcc4752ca4a1b80aabd38f6d19009ea5cbda0e0856983cf6d0023f5", - "sha256:c2dbb44c3f7e6c4d3487b31037b1bdbf424d97687c1747ce4ff2895795c9bf69", - "sha256:c79ebe8f3676a4c6630fd3f777f3cfecf9289666c84e775a67d1d358578dc2e3", - "sha256:c97528e64cb9ebeff9701e7938653a9951922f2a38bd847787d4a8e498cc83ae", - "sha256:d0611a0a2a518464c05ddd5a3a1a0e856ccc10e67079bb17f265ad19ab3c7597", - "sha256:d06adcfa62a4431d404c31216f0f8ac97397d799cd53800e9d3efc2fbb3cf14e", - "sha256:d41997519fcba4a1e46eb4a2fe31bc12f0ff957b2b81bac28db24744f333e955", - "sha256:d5b61785a9ce44e5a4b880272baa7cf6c8f48a5180c3e81c59553ba0cb0821ca", - "sha256:da152d8cdcab0e56e4f45eb08b9aea6455845ec83172092f09b0e077ece2cf7a", - "sha256:da7e547706e69e45d95e116e6939488d62174e033b763ab1496b4c29b76fabea", - "sha256:db5283d90da4174865d520e7366801a93777201e91e79bacbac6e6927cbceede", - "sha256:db608a6757adabb32f1cfe6066e39b3706d8c3aa69bbc353a5b61edad36a5cb4", - "sha256:e0ea21f66820452a3f5d1655f8704a60d66ba1191359b96541eaf457710a5fc6", - "sha256:e7da3fec7408813a7cebc9e4ec55afed2d0fd65c4754bc376bf03498d4e92686", - "sha256:e92a513161077b53447160b9bd8f522edfbed4bd9759e4c18ab05d7ef7e49408", - "sha256:ecb1fa0db7bf4cff9dac752abb19505a233c7f16684c5826d1f11ebd9472b871", - "sha256:efda5fc8cc1c61e4f639b8067d118e742b812c930f708e6667a5ce0d13499e29", - "sha256:f0a1dbdb5ecbef0d34eb77e56fcb3e95bbd7e50835d9782a45df81cc46949750", - "sha256:f0a71d85ecdd570ded8ac3d1c0f480842f49a40beb423bb8014539a9f32a5897", - "sha256:f4f270de01dd3e129a72efad823da90cc4d6aafb64c410c9033aba70db9f1ff0", - "sha256:f6cb459eea32a4e2cf18ba5fcece2dbdf496384413bc1bae15583f19e567f3b2", - "sha256:f8ad8285b01b0d4695102546b342b493b3ccc6781fc28c8c6a1bb63e95d22f09", - "sha256:f9f39e2f049db33a908319cf46624a569b36983c7c78318e9726a4cb8923b26c" + "sha256:00bd361b903dc4bbf4eb165f24d1acbee754fce22ded24c3d56eec268658a5cf", + "sha256:040c1aebeda72197ef477a906782b5ab0d387642e93bda547336b8957c61022e", + "sha256:05703cf211d585109fcd72207a31bb170a0f22144d68298dc5e61b3c946518af", + "sha256:06f54715b7737c2fecdbf140d1afb11a33d59508a47bf11bb38ecf21dc9ab79f", + "sha256:0dc9db8e79f0036e8173c466d21ef18e1befc02de8bf8aa8dc0813a6dc8a7046", + "sha256:0f114aa76dc1b8f636d077979c0ac22e7cd8f3493abbab152f20eb8d3cda71f3", + "sha256:11863aa14a51fd6ec28688d76f1735f8f69ab1fabf388851a595d0721af042f5", + "sha256:11c7de8f692fc99816e8ac50d1d1aef4f75126eefc33ac79aac02c099fd3db71", + "sha256:11d011a7574eb3b82bcc9c1a1d35c1d7075677fdd15de527d91b46bd35e935ee", + "sha256:146d14bebb7f1dc4d5fbf74f8a6cb15ac42baadee8912eb84ac0b3b2a3dc6ac3", + "sha256:15568384086b6df3c65353820a4473575dbad192e35010f622c6ce3eebd57af9", + "sha256:19df6e621f6d8b4b9c4d45f40a66839294ff2bb235e64d2178f7522d9170ac5b", + "sha256:1b04139c4236a0f3aff534479b58f6f849a8b351e1314826c2d230849ed48985", + "sha256:210ef2c3a1f03272649aff1ef992df2e724748918c4bc2d5a90352849eb40bea", + "sha256:2270953c0d8cdab5d422bee7d2007f043473f9d2999631c86a223c9db56cbd16", + "sha256:2400873bccc260b6ae184b2b8a4fec0e4082d30648eadb7c3d9a13405d861e89", + "sha256:2a40773c71d7ccdd3798f6489aaac9eee213d566850a9533f8d26332d626b82c", + "sha256:2c5674c4e74d939b9d91dda0fae10597ac7521768fec9e399c70a1f27e2ea2d9", + "sha256:3195782b26fc03aa9c6913d5bad5aeb864bdc372924c093b0f1cebad603dd712", + "sha256:31a82d498054cac9f6d0b53d02bb85811185bcb477d4b60144f915f3b3126342", + "sha256:32d5cf40c4f7c7b3ca500f8985eb3fb3a7dfc023215e876f207956b5ea26632a", + "sha256:346f5343b9e3f00b8db8ba359350eb124b98c99efd0b408728ac6ebf38173958", + "sha256:378a214a1e3bbf5ac4a8708304318b4f890da88c9e6a07699c4ae7174c09a68d", + "sha256:39b42c68602539407884cf70d6a480a469b93b81b7701378ba5e2328660c847a", + "sha256:3a2b053a0ab7a3960c98725cfb0bf5b48ba82f64ec95fe06f1d06c99b552e130", + "sha256:3aba7311af82e335dd1e36ffff68aaca609ca6290c2cb6d821a39aa075d8e3ff", + "sha256:3cd32d6c13807e5c66a7cbb79f90b553642f296ae4518a60d8d76243b0ad2898", + "sha256:3edd2fa14e68c9be82c5b16689e8d63d89fe927e56debd6e1dbce7a26a17f81b", + "sha256:4c380469bd3f970ef677bf2bcba2b6b0b4d5c75e7a020fb863ef75084efad66f", + "sha256:4e66e81a5779b65ac21764c295087de82235597a2293d18d943f8e9e32746265", + "sha256:53abb58632235cd154176ced1ae8f0d29a6657aa1aa9decf50b899b755bc2b93", + "sha256:5794cf59533bc3f1b1c821f7206a3617999db9fbefc345360aafe2e067514929", + "sha256:59415f46a37f7f2efeec758353dd2eae1b07640d8ca0f0c42548ec4125492635", + "sha256:59ec7b7c7e1a61061850d53aaf8e93db63dce0c936db1fda2658b70e4a1be709", + "sha256:59edc41b24031bc25108e210c0def6f6c2191210492a972d585a06ff246bb79b", + "sha256:5a580c91d686376f0f7c295357595c5a026e6cbc3d77b7c36e290201e7c11ecb", + "sha256:5b94529f9b2591b7af5f3e0e730a4e0a41ea174af35a4fd067775f9bdfeee01a", + "sha256:5c7b3b3a728dc6faf3fc372ef24f21d1e3cee2ac3e9596691d746e5a536de920", + "sha256:5c90ae8c8d32e472be041e76f9d2f2dbff4d0b0be8bd4041770eddb18cf49a4e", + "sha256:5e7139af55d1688f8b960ee9ad5adafc4ac17c1c473fe07133ac092310d76544", + "sha256:5ff5cf3571589b6d13bfbfd6bcd7a3f659e42f96b5fd1c4830c4cf21d4f5ef45", + "sha256:620ced262a86244e2be10a676b646f29c34537d0d9cc8eb26c08f53d98013390", + "sha256:6512cb89e334e4700febbffaaa52761b65b4f5a3cf33f960213d5656cea36a77", + "sha256:6c08e1312a9cf1074d17b17728d3dfce2a5125b2d791527f33ffbe805200a355", + "sha256:6c3bd3cde54cafb87d74d8db50b909705c62b17c2099b8f2e25b461882e544ff", + "sha256:6ef7afcd2d281494c0a9101d5c571970708ad911d028137cd558f02b851c08b4", + "sha256:7269d9e5f1084a653d575c7ec012ff57f0c042258bf5db0954bf551c158466e7", + "sha256:72d40b33e834371fd330fb1472ca19d9b8327acb79a5821d4008391db8e29f20", + "sha256:74d1b44c6cfc897df648cc9fdaa09bc3e7679926e6f96df05775d4fb3946571c", + "sha256:74db36e14a7d1ce0986fa104f7d5637aea5c82ca6326ed0ec5694280942d1162", + "sha256:763773d53f07244148ccac5b084da5adb90bfaee39c197554f01b286cf869228", + "sha256:76c6a5964640638cdeaa0c359382e5703e9293030fe730018ca06bc2010c4437", + "sha256:76d9289ed3f7501012e05abb8358bbb129149dbd173f1f57a1bf1c22d19ab7cc", + "sha256:7931d8f1f67c4be9ba1dd9c451fb0eeca1a25b89e4d3f89e828fe12a519b782a", + "sha256:7b8b454bac16428b22560d0a1cf0a09875339cab69df61d7805bf48919415901", + "sha256:7e5bab140c309cb3a6ce373a9e71eb7e4873c70c2dda01df6820474f9889d6d4", + "sha256:83d78376d0d4fd884e2c114d0621624b73d2aba4e2788182d286309ebdeed770", + "sha256:852542f9481f4a62dbb5dd99e8ab7aedfeb8fb6342349a181d4036877410f525", + "sha256:85267bd1aa8880a9c88a8cb71e18d3d64d2751a790e6ca6c27b8ccc724bcd5ad", + "sha256:88a2df29d4724b9237fc0c6eaf2a1adae0cdc0b3e9f4d8e7dc54b16812d2d81a", + "sha256:88b9f257ca61b838b6f8094a62418421f87ac2a1069f7e896c36a7d86b5d4c29", + "sha256:8ab3919a9997ab7ef2fbbed0cc99bb28d3c13e6d4b1ad36e97e482558a91be90", + "sha256:92dea1ffe3714fa8eb6a314d2b3c773208d865a0e0d35e713ec54eea08a66250", + "sha256:9407b6a5f0d675e8a827ad8742e1d6b49d9c1a1da5d952a67d50ef5f4170b18d", + "sha256:9408acf3270c4b6baad483865191e3e582b638b1654a007c62e3efe96f09a9a3", + "sha256:955e8513d07a283056b1396e9a57ceddbd272d9252c14f154d450d227606eb54", + "sha256:9db8ea4c388fdb0f780fe91346fd438657ea602d58348753d9fb265ce1bca67f", + "sha256:9eaa8b117dc8337728e834b9c6e2611f10c79e38f65157c4c38e9400286f5cb1", + "sha256:a51a263952b1429e429ff236d2f5a21c5125437861baeed77f5e1cc2d2c7c6da", + "sha256:a6aa6315319a052b4ee378aa171959c898a6183f15c1e541821c5c59beaa0238", + "sha256:aa12042de0171fad672b6c59df69106d20d5596e4f87b5e8f76df757a7c399aa", + "sha256:aaf7be1207676ac608a50cd08f102f6742dbfc70e8d60c4db1c6897f62f71523", + "sha256:b0157420efcb803e71d1b28e2c287518b8808b7cf1ab8af36718fd0a2c453eb0", + "sha256:b3f7e75f3015df442238cca659f8baa5f42ce2a8582727981cbfa15fee0ee205", + "sha256:b9098e0049e88c6a24ff64545cdfc50807818ba6c1b739cae221bbbcbc58aad3", + "sha256:ba55dce0a9b8ff59495ddd050a0225d58bd0983d09f87cfe2b6aec4f2c1234e4", + "sha256:bb86433b1cfe686da83ce32a9d3a8dd308e85c76b60896d58f082136f10bffac", + "sha256:bbea0db94288e29afcc4c28afbf3a7ccaf2d7e027489c449cf7e8f83c6346eb9", + "sha256:bbf1d63eef84b2e8c89011b7f2235b1e0bf7dacc11cac9431fc6468e99ac77fb", + "sha256:c7940c1dc63eb37a67721b10d703247552416f719c4188c54e04334321351ced", + "sha256:c9bf3325c47b11b2e51bca0824ea217c7cd84491d8ac4eefd1e409705ef092bd", + "sha256:cdc8a402aaee9a798b50d8b827d7ecf75edc5fb35ea0f91f213ff927c15f4ff0", + "sha256:ceec1a6bc6cab1d6ff5d06592a91a692f90ec7505d6463a88a52cc0eb58545da", + "sha256:cfe6ab8da05c01ba6fbea630377b5da2cd9bcbc6338510116b01c1bc939a2c18", + "sha256:d099e745a512f7e3bbe7249ca835f4d357c586d78d79ae8f1dcd4d8adeb9bda9", + "sha256:d0ef46024e6a3d79c01ff13801cb19d0cad7fd859b15037aec74315540acc276", + "sha256:d2e5a98f0ec99beb3c10e13b387f8db39106d53993f498b295f0c914328b1333", + "sha256:da4cfb373035def307905d05041c1d06d8936452fe89d464743ae7fb8371078b", + "sha256:da802a19d6e15dffe4b0c24b38b3af68e6c1a68e6e1d8f30148c83864f3881db", + "sha256:dced8146011d2bc2e883f9bd68618b8247387f4bbec46d7392b3c3b032640126", + "sha256:dfdd7c0b105af050eb3d64997809dc21da247cf44e63dc73ff0fd20b96be55a9", + "sha256:e368f200bbc2e4f905b8e71eb38b3c04333bddaa6a2464a6355487b02bb7fb09", + "sha256:e391b1f0a8a5a10ab3b9bb6afcfd74f2175f24f8975fb87ecae700d1503cdee0", + "sha256:e57e563a57fb22a142da34f38acc2fc1a5c864bc29ca1517a88abc963e60d6ec", + "sha256:e5d706eba36b4c4d5bc6c6377bb6568098765e990cfc21ee16d13963fab7b3e7", + "sha256:ec20916e7b4cbfb1f12380e46486ec4bcbaa91a9c448b97023fde0d5bbf9e4ff", + "sha256:f1d072c2eb0ad60d4c183f3fb44ac6f73fb7a8f16a2694a91f988275cbf352f9", + "sha256:f846c260f483d1fd217fe5ed7c173fb109efa6b1fc8381c8b7552c5781756192", + "sha256:f91de7223d4c7b793867797bacd1ee53bfe7359bd70d27b7b58a04efbb9436c8", + "sha256:faae4860798c31530dd184046a900e652c95513796ef51a12bc086710c2eec4d", + "sha256:fc579bf0f502e54926519451b920e875f433aceb4624a3646b3252b5caa9e0b6", + "sha256:fcc700eadbbccbf6bc1bcb9dbe0786b4b1cb91ca0dcda336eef5c2beed37b797", + "sha256:fd32ea360bcbb92d28933fc05ed09bffcb1704ba3fc7942e81db0fd4f81a7892", + "sha256:fdb7adb641a0d13bdcd4ef48e062363d8a9ad4a182ac7647ec88f695e719ae9f" ], "markers": "python_version >= '3.7'", - "version": "==1.4.4" + "version": "==1.4.5" }, "mapclassify": { "hashes": [ @@ -1749,28 +1785,34 @@ }, "scipy": { "hashes": [ - "sha256:08d957ca82d3535b3b9ba6c8ff355d78fe975271874e2af267cb5add5bd78625", - "sha256:249cfa465c379c9bb2c20123001e151ff5e29b351cbb7f9c91587260602c58d0", - "sha256:366a6a937110d80dca4f63b3f5b00cc89d36f678b2d124a01067b154e692bab1", - "sha256:39154437654260a52871dfde852adf1b93b1d1bc5dc0ffa70068f16ec0be2624", - "sha256:396fae3f8c12ad14c5f3eb40499fd06a6fef8393a6baa352a652ecd51e74e029", - "sha256:3b9963798df1d8a52db41a6fc0e6fa65b1c60e85d73da27ae8bb754de4792481", - "sha256:3e8eb42db36526b130dfbc417609498a6192381abc1975b91e3eb238e0b41c1a", - "sha256:512fdc18c65f76dadaca139348e525646d440220d8d05f6d21965b8d4466bccd", - "sha256:aec8c62fbe52914f9cf28d846cf0401dd80ab80788bbab909434eb336ed07c04", - "sha256:b41a0f322b4eb51b078cb3441e950ad661ede490c3aca66edef66f4b37ab1877", - "sha256:b4bb943010203465ac81efa392e4645265077b4d9e99b66cf3ed33ae12254173", - "sha256:b588311875c58d1acd4ef17c983b9f1ab5391755a47c3d70b6bd503a45bfaf71", - "sha256:ba94eeef3c9caa4cea7b402a35bb02a5714ee1ee77eb98aca1eed4543beb0f4c", - "sha256:be8c962a821957fdde8c4044efdab7a140c13294997a407eaee777acf63cbf0c", - "sha256:cce154372f0ebe88556ed06d7b196e9c2e0c13080ecb58d0f35062dc7cc28b47", - "sha256:d51565560565a0307ed06fa0ec4c6f21ff094947d4844d6068ed04400c72d0c3", - "sha256:e866514bc2d660608447b6ba95c8900d591f2865c07cca0aa4f7ff3c4ca70f30", - "sha256:fb5b492fa035334fd249f0973cc79ecad8b09c604b42a127a677b45a9a3d4289", - "sha256:ffb28e3fa31b9c376d0fb1f74c1f13911c8c154a760312fbee87a21eb21efe31" + "sha256:0f3261f14b767b316d7137c66cc4f33a80ea05841b9c87ad83a726205b901423", + "sha256:10eb6af2f751aa3424762948e5352f707b0dece77288206f227864ddf675aca0", + "sha256:1342ca385c673208f32472830c10110a9dcd053cf0c4b7d4cd7026d0335a6c1d", + "sha256:214cdf04bbae7a54784f8431f976704ed607c4bc69ba0d5d5d6a9df84374df76", + "sha256:2b997a5369e2d30c97995dcb29d638701f8000d04df01b8e947f206e5d0ac788", + "sha256:2c91cf049ffb5575917f2a01da1da082fd24ed48120d08a6e7297dfcac771dcd", + "sha256:3aeb87661de987f8ec56fa6950863994cd427209158255a389fc5aea51fa7055", + "sha256:4447ad057d7597476f9862ecbd9285bbf13ba9d73ce25acfa4e4b11c6801b4c9", + "sha256:542a757e2a6ec409e71df3d8fd20127afbbacb1c07990cb23c5870c13953d899", + "sha256:8d9886f44ef8c9e776cb7527fb01455bf4f4a46c455c4682edc2c2cc8cd78562", + "sha256:90d3b1364e751d8214e325c371f0ee0dd38419268bf4888b2ae1040a6b266b2a", + "sha256:95763fbda1206bec41157582bea482f50eb3702c85fffcf6d24394b071c0e87a", + "sha256:ac74b1512d38718fb6a491c439aa7b3605b96b1ed3be6599c17d49d6c60fca18", + "sha256:afdb0d983f6135d50770dd979df50bf1c7f58b5b33e0eb8cf5c73c70600eae1d", + "sha256:b0620240ef445b5ddde52460e6bc3483b7c9c750275369379e5f609a1050911c", + "sha256:b133f237bd8ba73bad51bc12eb4f2d84cbec999753bf25ba58235e9fc2096d80", + "sha256:b29318a5e39bd200ca4381d80b065cdf3076c7d7281c5e36569e99273867f61d", + "sha256:b8425fa963a32936c9773ee3ce44a765d8ff67eed5f4ac81dc1e4a819a238ee9", + "sha256:d2b813bfbe8dec6a75164523de650bad41f4405d35b0fa24c2c28ae07fcefb20", + "sha256:d690e1ca993c8f7ede6d22e5637541217fc6a4d3f78b3672a6fe454dbb7eb9a7", + "sha256:e367904a0fec76433bf3fbf3e85bf60dae8e9e585ffd21898ab1085a29a04d16", + "sha256:ea932570b1c2a30edafca922345854ff2cd20d43cd9123b6dacfdecebfc1a80b", + "sha256:f28f1f6cfeb48339c192efc6275749b2a25a7e49c4d8369a28b6591da02fbc9a", + "sha256:f73102f769ee06041a3aa26b5841359b1a93cc364ce45609657751795e8f4a4a", + "sha256:fa4909c6c20c3d91480533cddbc0e7c6d849e7d9ded692918c76ce5964997898" ], "markers": "python_version < '3.13' and python_version >= '3.9'", - "version": "==1.11.1" + "version": "==1.11.2" }, "send2trash": { "hashes": [ @@ -1782,11 +1824,11 @@ }, "setuptools": { "hashes": [ - "sha256:d59c97e7b774979a5ccb96388efc9eb65518004537e85d52e81eaee89ab6dd91", - "sha256:e13e1b0bc760e9b0127eda042845999b2f913e12437046e663b833aa96d89715" + "sha256:3d4dfa6d95f1b101d695a6160a7626e15583af71a5f52176efa5d39a054d475d", + "sha256:3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b" ], "markers": "python_version >= '3.8'", - "version": "==68.1.0" + "version": "==68.1.2" }, "shapely": { "hashes": [ @@ -2020,11 +2062,11 @@ }, "websocket-client": { "hashes": [ - "sha256:c951af98631d24f8df89ab1019fc365f2227c0892f12fd150e935607c79dd0dd", - "sha256:f1f9f2ad5291f0225a49efad77abf9e700b6fef553900623060dad6e26503b9d" + "sha256:53e95c826bf800c4c465f50093a8c4ff091c7327023b10bfaff40cf1ef170eaa", + "sha256:ce54f419dfae71f4bdba69ebe65bf7f0a93fe71bc009ad3a010aacc3eebad537" ], - "markers": "python_version >= '3.7'", - "version": "==1.6.1" + "markers": "python_version >= '3.8'", + "version": "==1.6.2" }, "widgetsnbextension": { "hashes": [ diff --git a/README.md b/README.md index 4aa9406b..22653a2f 100644 --- a/README.md +++ b/README.md @@ -21,10 +21,15 @@ Gun crime x vacant lots mapping tool layer `CREATE EXTENSION postgis;` 5. Set your database connection string to an environment variable `VACANT_LOTS_DB` +# awkde + +We are using the [awkde package](https://github.com/mennthor/awkde) to create the Adaptive Width KDE. It is not available through pip. Install directly in the `src` folder using their instructions. + # Usage Instructions 1. Activate the pipenv shell: `pipenv shell` -2. Run the main script: `python process_data.py [output_filename]` +2. Move the to src folder `cd src` +3. Run the main script `python script.py` # Contribution Instructions diff --git a/archived/constants.py b/archived/constants.py deleted file mode 100644 index c31082d6..00000000 --- a/archived/constants.py +++ /dev/null @@ -1,97 +0,0 @@ -ARCGIS_BASE_URL: str = "https://services.arcgis.com" - -ARCGIS_LAND_DATASET_PATH: str = "/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/Vacant_Indicators_Land/FeatureServer/0/query" -ARCGIS_BUILDINGS_DATASET_PATH: str = "/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/Vacant_Indicators_Bldg/FeatureServer/0/query" -ARCGIS_PHS_LANDCARE_DATASET_PATH: str = ( - "/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/PHS_CommunityLandcare/FeatureServer/0/query" -) -ARCGIS_PHS_MAINTENANCE_DATASET_PATH: str = "/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/PHS_PhilaLandCare_Maintenance/FeatureServer/0/query" -ARCGIS_COMMUNITY_ORGANISATIONS_DATASET_PATH: str = ( - "/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/Zoning_RCO/FeatureServer/0/query" -) - -PHILADELPHIA_CARTO_BASE_URL: str = "https://phl.carto.com/api/v2/sql" - -PHILADELPHIA_NEIGHBORHOODS_DATASET_URL: str = "https://github.com/azavea/geo-data/raw/master/Neighborhoods_Philadelphia/Neighborhoods_Philadelphia.zip" - -PHILADELPHIA_TREE_CANOPY_DATASET_URL: str = ( - "https://national-tes-data-share.s3.amazonaws.com/national_tes_share/pa.zip.zip" -) - -DEFAULT_ARCGIS_QUERY_PARAMETERS: dict = { - "where": "1=1", - "outFields": "*", - "returnGeometry": "true", - "f": "json", - "resultRecordCount": 2000, -} - -PUBLIC_PROPERTY_OWNER_VALUES = [ - "PHILADELPHIA HOUSING DEVELOPMENT CORPORATION", - "REDEVELOPMENT AUTHORITY OF PHILADELPHIA", - "PHILA REDEVELOPMENT AUTHO", - "COMMONWEALTH OF PENNA", - "DEPT OF PUBLIC PROP; CITY OF PHILA", - "PHILA HOUSING AUTHORITY", - "PHILADELPHIA HOUSING AUTH", - "PUBLIC PROP DIV; CITY OF PHILA", - "COMMONWEALTH OF PA", - "REAL ESTATE DIV; CITY OF PHILA", - "PHILADELPHIA HOUSING AUTHORITY", - "PHILADELPHIA REDEVELOPMENT AUTH", - "DEVELOPMENT CORPORATION; PHILADELPHIA HOUSING", - "DEPT OF PUB PROP; CITY OF PHILA", - "URBAN DEVELOPMENT; SECRETARY OF HOUSINGURBAN DEVELOPMENT; SECRETARY OF HOUSING AND", - "PHILADELPHIA LAND BANK", - "PUBLIC PROP REAL ESTATE; CITY OF PHILA", - "PENNDOT", - "REAL ESTATE DIVISION; CITY OF PHILA", - "PHILADELPHIA LAND INVESTM", - "DEPT OF PUBLIC PROP; CITY OF PHILADELPHIA", - "REDEVELOPMENT AUTHORITY OF PHILA", - "CITY OF PHILADELPHIA DEPARTMENT OF PUBLIC PROPERTY", - "DEPT OF PUBLC PROP; CITY OF PHILA", - "PHILA HOUSING DEV CORP", - "PHILA REDEVELOPMENT AUTH", - "KENSINGTON HOUSING AUTHOR", - "REDEVELOPMENT AUTHORITY O", - "PHILADELPHIA LAND INVESTMENT", - "PHILADELPHIA REDEVELOPMEN", - "COMMONWEALTH OF PENNSYLVANIA", - "DEP OF PUB PROP; CITY OF PHILA", - "SECRETARY OF HOUSING AND URBAN DEVELOPMENT", - "PHILADELPHIA REDEVELOPMENT AUTHORITY", - "CITY OF PHILADELPHIA", - "KENSINGTON HOUSING AUTHORITY", - "DEPT PUBLIC PROP R E DIV; CITY OF PHILA", - "REDEVELOPMENT AUTHORITY", - "CITY OF PHILA", - "DEPT PUB PROP; CITY OF PHILA", -] - -REDEVELOPMENT_OWNER_VALUES = [ - "REDEVELOPMENT AUTHORITY OF PHILA", - "PHILA REDEVELOPMENT AUTH", - "REDEVELOPMENT AUTHORITY O", - "PHILADELPHIA REDEVELOPMEN", - "PHILA REDEVELOPMENT AUTHO", - "REDEVELOPMENT AUTHORITY", - "REDEVELOPMENT AUTH", -] - -DEPARTMENT_OF_PUBLIC_PROPERTY_OWNER_VALUES = [ - "DEPT OF PUBLIC PROPERTY", - "DEPT OF PUBLIC PROPERT", - "DEPT OF PUBLC PROP; CITY OF " "PHILA", - "DEPT OF PUBLIC PROP; CITY OF PHILA", - "DEPT OF PUBLIC PROPERTY; CITY OF PHILA", - "DEPT PUB PROP; CITY OF PHILA", - "DEPT OF PUB PROP; CITY OF PHILA", - "DEP OF PUB PROP; CITY OF PHILA", - "DEPT OF PUBLIC PROP; CITY OF PHILADELPHIA", - "DEPT PUBLIC PROP R E DIV; CITY OF PHILA", - "PUBLIC PROP DIV; CITY OF PHILA", - "PUBLIC PROP REAL ESTATE; CITY OF PHILA", - "REAL ESTATE DIV; CITY OF PHILA", - "REAL ESTATE DIVISION; CITY OF PHILA", -] diff --git a/archived/data_utils.py b/archived/data_utils.py deleted file mode 100644 index 82ddcb40..00000000 --- a/archived/data_utils.py +++ /dev/null @@ -1,248 +0,0 @@ -import datetime -import io -import warnings - -import fiona -import geopandas as gpd -import pandas as pd -import requests -from shapely import Polygon - -from constants import ( - DEFAULT_ARCGIS_QUERY_PARAMETERS, - ARCGIS_BASE_URL, - PHILADELPHIA_CARTO_BASE_URL, - REDEVELOPMENT_OWNER_VALUES, - DEPARTMENT_OF_PUBLIC_PROPERTY_OWNER_VALUES, - PUBLIC_PROPERTY_OWNER_VALUES, -) - -warnings.filterwarnings("ignore", "GeoSeries.notna", UserWarning) - - -def get_arcgis_dataset(dataset_path: str) -> gpd.GeoDataFrame: - dfs: list[pd.DataFrame] = [] - pagination_offset: int = 0 - - while True: - query_params = DEFAULT_ARCGIS_QUERY_PARAMETERS | { - "resultOffset": pagination_offset - } - response = requests.get(url=ARCGIS_BASE_URL + dataset_path, params=query_params) - if response.status_code != 200: - print( - f"Error calling ArcGIS API, request returned HTTP {response.status_code}" - ) - break - - df = pd.DataFrame(response.json()["features"]) - - # separate the attributes column into one column per attribute - df = pd.concat( - [df.drop(["attributes"], axis=1), df["attributes"].apply(pd.Series)], axis=1 - ) - - # make the `geometry` column a shapely geometry object - df["geometry"] = df["geometry"].apply(lambda x: Polygon(x["rings"][0])) - - land_gdf = gpd.GeoDataFrame(df, geometry="geometry", crs="EPSG:3857") - dfs.append(land_gdf.to_crs(epsg=2272)) - - pagination_offset += len(land_gdf) - - # we specify 2000 results in the query, <2000 indicates no further results - if len(land_gdf) < 2000: - break - - return gpd.GeoDataFrame(pd.concat(dfs, ignore_index=True), crs="EPSG:2272") - - -def get_philadelphia_li_complaint_dataset() -> gpd.GeoDataFrame: - one_year_ago: str = ( - datetime.datetime.now() - datetime.timedelta(days=365) - ).strftime("%Y-%m-%d") - - query: str = ( - f"SELECT address, service_request_id, subject, status, service_name, service_code, lat, lon" - f" FROM public_cases_fc " - f"WHERE requested_datetime >= '{one_year_ago}'" - ) - - response = requests.get(url=PHILADELPHIA_CARTO_BASE_URL, params={"q": query}) - - df = pd.DataFrame(response.json()["rows"]) - return gpd.GeoDataFrame( - df, geometry=gpd.points_from_xy(df.lon, df.lat), crs="EPSG:2272" - ) - - -def get_philadelphia_li_violation_dataset() -> gpd.GeoDataFrame: - one_year_ago: str = ( - datetime.datetime.now() - datetime.timedelta(days=365) - ).strftime("%Y-%m-%d") - - query: str = ( - f"SELECT parcel_id_num, casenumber, casecreateddate, casetype, casestatus, violationnumber," - f" violationcodetitle, violationstatus, opa_account_num, address, opa_owner, " - f" geocode_x, geocode_y " - f"FROM violations " - f"WHERE violationdate >= '{one_year_ago}'" - ) - - response = requests.get(url=PHILADELPHIA_CARTO_BASE_URL, params={"q": query}) - - df = pd.DataFrame(response.json()["rows"]) - - return gpd.GeoDataFrame( - df, - geometry=gpd.points_from_xy(df.geocode_x, df.geocode_y), - crs="EPSG:2272", - ) - - -def get_philadelphia_gun_crime_dataset() -> gpd.GeoDataFrame: - one_year_ago: str = ( - datetime.datetime.now() - datetime.timedelta(days=365) - ).strftime("%Y-%m-%d") - - query = ( - f"SELECT text_general_code, dispatch_date, point_x, point_y " - f"FROM incidents_part1_part2 " - f"WHERE dispatch_date_time >= '{one_year_ago}' " - f"AND text_general_code IN ('Aggravated Assault Firearm', 'Robbery Firearm')" - ) - - response = requests.get(url=PHILADELPHIA_CARTO_BASE_URL, params={"q": query}) - - df = pd.DataFrame(response.json()["rows"]) - - gdf = gpd.GeoDataFrame( - df, geometry=gpd.points_from_xy(df.point_x, df.point_y), crs="EPSG:4326" - ) - - gdf.drop(["point_x", "point_y"], axis=1, inplace=True) - - gdf.to_crs(epsg=2272, inplace=True) - - return gdf[gdf["geometry"].notnull()] - - -def get_property_assessment_dataset() -> pd.DataFrame: - query = "SELECT parcel_number, market_value " "FROM opa_properties_public" - - response = requests.get(url=PHILADELPHIA_CARTO_BASE_URL, params={"q": query}) - - return pd.DataFrame(response.json()["rows"]) - - -def get_delinquency_dataset() -> pd.DataFrame: - query = "SELECT * FROM real_estate_tax_delinquencies" - - response = requests.get(url=PHILADELPHIA_CARTO_BASE_URL, params={"q": query}) - - return pd.DataFrame(response.json()["rows"]) - - -def get_shapefile_dataset(dataset_url: str) -> gpd.GeoDataFrame: - response = requests.get(url=dataset_url) - - bytes_array = io.BytesIO(response.content).read() - - with fiona.BytesCollection(bytes_array) as src: - crs = src.crs - gdf = gpd.GeoDataFrame.from_features(src, crs=crs) - - return gdf - - -def starts_with_preposition(string): - return string.split(" ")[0].lower() in [ - "a", - "an", - "and", - "as", - "at", - "but", - "by", - "for", - "from", - "in", - "into", - "nor", - "of", - "on", - "or", - "so", - "the", - "to", - "up", - "yet", - ] - - -def combine_owners(row): - if pd.isnull(row["OWNER1"]) and pd.isnull(row["OWNER2"]): - return None - elif pd.isnull(row["OWNER1"]) and not pd.isnull(row["OWNER2"]): - return row["OWNER2"] - elif not pd.isnull(row["OWNER1"]) and pd.isnull(row["OWNER2"]): - return row["OWNER1"] - elif starts_with_preposition(row["OWNER2"]): - return row["OWNER1"] + " " + row["OWNER2"] - else: - return row["OWNER2"] + "; " + row["OWNER1"] - - -def clean_ownership_values(gdf: gpd.GeoDataFrame) -> gpd.GeoDataFrame: - """ - correct misspellings, and identify public vs. private ownership - """ - gdf["OWNER"] = gdf.apply(combine_owners, axis=1) - - gdf.loc[ - gdf["OWNER"] == "PHILADELPHIA HOUSING AUTH", "OWNER" - ] = "PHILADELPHIA HOUSING AUTHORITY" - gdf.loc[ - gdf["OWNER"] == "PHILA HOUSING AUTHORITY", "OWNER" - ] = "PHILADELPHIA HOUSING AUTHORITY" - - for val in REDEVELOPMENT_OWNER_VALUES: - gdf.loc[ - gdf["OWNER"] == val, "OWNER" - ] = "REDEVELOPMENT AUTHORITY OF PHILADELPHIA" - - for var in DEPARTMENT_OF_PUBLIC_PROPERTY_OWNER_VALUES: - gdf.loc[ - gdf["OWNER"] == var, "OWNER" - ] = "CITY OF PHILADELPHIA DEPARTMENT OF PUBLIC PROPERTY" - - gdf.loc[ - gdf["OWNER"] == "URBAN DEVELOPMENT; SECRETARY OF HOUSING", "OWNER" - ] = "SECRETARY OF HOUSING AND URBAN DEVELOPMENT" - gdf.loc[ - gdf["OWNER"] == "URBAN DEVELOPMENT; SECRETARY OF HOUSING AND", "OWNER" - ] = "SECRETARY OF HOUSING AND URBAN DEVELOPMENT" - - gdf.loc[ - gdf["OWNER"] == "COMMONWEALTH OF PA", "OWNER" - ] = "COMMONWEALTH OF PENNSYLVANIA" - gdf.loc[ - gdf["OWNER"] == "COMMONWEALTH OF PENNA", "OWNER" - ] = "COMMONWEALTH OF PENNSYLVANIA" - - gdf.loc[ - gdf["OWNER"] == "DEVELOPMENT CORPORATION; PHILADELPHIA HOUSING", "OWNER" - ] = "PHILADELPHIA HOUSING DEVELOPMENT CORPORATION" - gdf.loc[ - gdf["OWNER"] == "PHILA HOUSING DEV CORP", "OWNER" - ] = "PHILADELPHIA HOUSING DEVELOPMENT CORPORATION" - - gdf.loc[ - gdf["OWNER"] == "DEPARTMENT OF TRANSPORTAT; COMMONWEALTH OF PENNSYLVA", "OWNER" - ] = "PENNDOT" - - gdf.loc[gdf["OWNER"] == "CITY OF PHILADELPHIA", "OWNER"] = "CITY OF PHILA" - - gdf["public_owner"] = gdf["OWNER"].isin(PUBLIC_PROPERTY_OWNER_VALUES) - - return gdf.drop(["OWNER1", "OWNER2"], axis=1) diff --git a/archived/process_data.py b/archived/process_data.py deleted file mode 100644 index 106da7b4..00000000 --- a/archived/process_data.py +++ /dev/null @@ -1,267 +0,0 @@ -import argparse -import os - -import geopandas as gpd -import pandas as pd -import logging - -from constants import ( - ARCGIS_LAND_DATASET_PATH, - ARCGIS_BUILDINGS_DATASET_PATH, - ARCGIS_PHS_LANDCARE_DATASET_PATH, - ARCGIS_PHS_MAINTENANCE_DATASET_PATH, - ARCGIS_COMMUNITY_ORGANISATIONS_DATASET_PATH, - PHILADELPHIA_TREE_CANOPY_DATASET_URL, - PHILADELPHIA_NEIGHBORHOODS_DATASET_URL, -) -from data_utils import ( - get_arcgis_dataset, - get_philadelphia_li_complaint_dataset, - get_philadelphia_li_violation_dataset, - get_shapefile_dataset, - get_philadelphia_gun_crime_dataset, - get_property_assessment_dataset, - get_delinquency_dataset, - clean_ownership_values, -) - -logger = logging.getLogger(__name__) -logging.basicConfig( - level=os.environ.get("LOGLEVEL", "INFO"), format="%(asctime)s %(message)s" -) - - -def clean_and_merge_gdfs( - land_gdf, - buildings_gdf, - phs_landcare_gdf, - phs_maintenance_gdf, - community_orgs_gdf, - complaints_gdf, - violations_gdf, - neighborhoods_gdf, - tree_canopy_gdf, - gun_crime_gdf, - property_assessment_df, - delinquency_df, -) -> gpd.GeoDataFrame: - """this method needs quite a bit of refactoring""" - - """ land & buildings """ - columns = [ - "geometry", - "ADDRESS", - "BLDG_DESC", - "OPA_ID", - "COUNCILDISTRICT", - "ZIPCODE", - "OWNER1", - "OWNER2", - ] - - land_gdf, buildings_gdf = land_gdf[columns], buildings_gdf[columns] - land_gdf.insert(0, "type", "lot") - buildings_gdf.insert(0, "type", "building") - - main_gdf = pd.concat([land_gdf, buildings_gdf], axis=0) - main_gdf = clean_ownership_values(main_gdf) - - """ PHS """ - phs_landcare_gdf = phs_landcare_gdf[["geometry", "COMM_PARTN"]] - phs_maintenance_gdf.insert(0, "COMM_PARTN", "PHS") - phs_maintenance_gdf = phs_maintenance_gdf[["geometry", "COMM_PARTN"]] - - phs_gdf = gpd.GeoDataFrame( - pd.concat([phs_landcare_gdf, phs_maintenance_gdf], ignore_index=True), - crs=phs_landcare_gdf.crs, - ) - - main_gdf = gpd.sjoin(main_gdf, phs_gdf, how="left", predicate="intersects") - main_gdf.drop(["index_right"], axis=1, inplace=True) - - main_gdf["COMM_PARTN"] = main_gdf["COMM_PARTN"].fillna("None", inplace=True) - - """ L&I """ - complaints_gdf.drop(["lat", "lon"], axis=1, inplace=True) - - complaints_gdf = complaints_gdf[complaints_gdf["status"] == "Open"] - - complaints_gdf = ( - complaints_gdf.groupby("address")["service_name"] - .apply(lambda x: "; ".join([val for val in x if val is not None])) - .reset_index() - ) - - complaints_gdf.rename(columns={"service_name": "li_complaints"}, inplace=True) - - all_violations_count_df = ( - violations_gdf.groupby("opa_account_num") - .count() - .reset_index()[["opa_account_num", "violationnumber"]] - ) - all_violations_count_df = all_violations_count_df.rename( - columns={"violationnumber": "all_violations_past_year"} - ) - - violations_gdf = violations_gdf[(violations_gdf["violationstatus"] == "OPEN")] - open_violations_count_df = ( - violations_gdf.groupby("opa_account_num") - .count() - .reset_index()[["opa_account_num", "violationnumber"]] - ) - open_violations_count_df = open_violations_count_df.rename( - columns={"violationnumber": "open_violations_past_year"} - ) - - violations_count_gdf = all_violations_count_df.merge( - open_violations_count_df, how="left", on="opa_account_num" - ) - violations_count_gdf.fillna(0, inplace=True) - - violations_count_gdf["all_violations_past_year"] = violations_count_gdf[ - "all_violations_past_year" - ].astype(int) - violations_count_gdf["open_violations_past_year"] = violations_count_gdf[ - "open_violations_past_year" - ].astype(int) - - violations_gdf = ( - violations_gdf.groupby("opa_account_num")["violationcodetitle"] - .apply(lambda x: "; ".join([val for val in x if val is not None])) - .reset_index() - ) - violations_gdf.rename( - columns={"violationcodetitle": "li_code_violations"}, inplace=True - ) - - main_gdf = main_gdf.merge( - complaints_gdf, how="left", left_on="ADDRESS", right_on="address" - ) - main_gdf = main_gdf.merge( - violations_gdf, how="left", left_on="OPA_ID", right_on="opa_account_num" - ) - - main_gdf.drop(["address", "opa_account_num"], axis=1, inplace=True) - - """ Centroids """ - main_gdf["centroid"] = main_gdf["geometry"].centroid - poly_gdf = main_gdf[["OPA_ID", "geometry"]] - main_gdf.drop(["geometry"], axis=1, inplace=True) - main_gdf.set_geometry("centroid", inplace=True) - - """ neighborhoods """ - main_gdf = gpd.sjoin( - main_gdf, neighborhoods_gdf, how="left", predicate="intersects" - ) - main_gdf = main_gdf.drop( - ["index_right", "NAME", "LISTNAME", "Shape_Leng", "Shape_Area"], axis=1 - ) - main_gdf.rename(columns={"MAPNAME": "neighborhood"}, inplace=True) - - """ community orgs """ - community_orgs_gdf["PRIMARY_PHONE"] = community_orgs_gdf["PRIMARY_PHONE"].astype( - str - ) - community_orgs_gdf["EXPIRATIONYEAR"] = community_orgs_gdf["EXPIRATIONYEAR"].astype( - str - ) - rco_aggregate_cols = [ - "ORGANIZATION_NAME", - "ORGANIZATION_ADDRESS", - "PRIMARY_EMAIL", - "PRIMARY_PHONE", - ] - community_orgs_gdf["rco_info"] = community_orgs_gdf[rco_aggregate_cols].agg( - "; ".join, axis=1 - ) - rcos_final_cols = ["geometry", "rco_info"] - community_orgs_gdf = community_orgs_gdf[rcos_final_cols] - - w_community_orgs_gdf = gpd.sjoin( - main_gdf, community_orgs_gdf, how="left", predicate="within" - ) - w_community_orgs_gdf.drop(["index_right"], axis=1, inplace=True) - w_community_orgs_gdf.rename(columns={"rco_info": "rco_info"}, inplace=True) - w_community_orgs_gdf.drop_duplicates(subset="OPA_ID", inplace=True) - - w_community_orgs_gdf["OPA_ID"] = w_community_orgs_gdf["OPA_ID"].astype(str) - - rcos_by_opa_id_gdf = ( - w_community_orgs_gdf.groupby("OPA_ID")["rco_info"] - .apply(lambda x: "| ".join([str(val) for val in x if val is not None])) - .reset_index() - ) - - rcos_by_opa_id_gdf.rename(columns={"rco_info": "relevant_rcos"}, inplace=True) - - main_gdf = main_gdf.merge( - rcos_by_opa_id_gdf, how="left", left_on="OPA_ID", right_on="OPA_ID" - ) - main_gdf.drop_duplicates(subset="OPA_ID", inplace=True) - - """ tree canopy """ - main_gdf = gpd.sjoin(main_gdf, tree_canopy_gdf, how="left", predicate="intersects") - main_gdf.drop(["index_right"], axis=1, inplace=True) - main_gdf.drop_duplicates(subset="OPA_ID", inplace=True) - - return main_gdf - - -def orchestrate_processing(output_filename: str) -> bool: - logger.info("Fetching data sources") - land_gdf = get_arcgis_dataset(ARCGIS_LAND_DATASET_PATH) - buildings_gdf = get_arcgis_dataset(ARCGIS_BUILDINGS_DATASET_PATH) - phs_landcare_gdf = get_arcgis_dataset(ARCGIS_PHS_LANDCARE_DATASET_PATH) - phs_maintenance_gdf = get_arcgis_dataset(ARCGIS_PHS_MAINTENANCE_DATASET_PATH) - community_orgs_gdf = get_arcgis_dataset(ARCGIS_COMMUNITY_ORGANISATIONS_DATASET_PATH) - complaints_gdf = get_philadelphia_li_complaint_dataset() - violations_gdf = get_philadelphia_li_violation_dataset() - neighborhoods_gdf = get_shapefile_dataset(PHILADELPHIA_NEIGHBORHOODS_DATASET_URL) - tree_canopy_gdf = get_shapefile_dataset(PHILADELPHIA_TREE_CANOPY_DATASET_URL) - gun_crime_gdf = get_philadelphia_gun_crime_dataset() - property_assessment_df = get_property_assessment_dataset() - delinquency_df = get_delinquency_dataset() - logger.info("Successfully fetched data") - - logger.info("Cleaning and merging the data") - main_gdf = clean_and_merge_gdfs( - land_gdf, - buildings_gdf, - phs_landcare_gdf, - phs_maintenance_gdf, - community_orgs_gdf, - complaints_gdf, - violations_gdf, - neighborhoods_gdf, - tree_canopy_gdf, - gun_crime_gdf, - property_assessment_df, - delinquency_df, - ) - logger.info("Successfully processed data") - - logger.info(f"Writing data to ./{output_filename}") - main_gdf.to_file(f"./{output_filename}", driver="GeoJSON") - - return True - - -def main(): - parser: argparse.ArgumentParser = argparse.ArgumentParser( - description="Process various Philadelphia data into a GeoJSON file" - ) - parser.add_argument("filename", help="the output file name") - args = parser.parse_args() - - output_filename: str = args.filename - - if orchestrate_processing(output_filename): - logger.info( - f"Successfully processed data. Results are saved in {output_filename}" - ) - else: - logger.info("An error occurred during data processing.") - - -if __name__ == "__main__": - main() diff --git a/notes.md b/notes.md deleted file mode 100644 index aa015d88..00000000 --- a/notes.md +++ /dev/null @@ -1 +0,0 @@ -L&I complaints- merge by intersect instead of on address diff --git a/src/classes/featurelayer.py b/src/classes/featurelayer.py index 5d3a9dfd..fb87d6d9 100644 --- a/src/classes/featurelayer.py +++ b/src/classes/featurelayer.py @@ -39,24 +39,22 @@ def __init__( inputs = [self.esri_rest_urls, self.carto_sql_queries, self.gdf] non_none_inputs = [i for i in inputs if i is not None] - if len(non_none_inputs) != 1: - raise ValueError( - "Exactly one of esri_rest_urls, carto_sql_queries, or gdf must be provided." - ) - - if self.esri_rest_urls is not None: - self.type = "esri" - elif self.carto_sql_queries is not None: - self.type = "carto" - elif self.gdf is not None: - self.type = "gdf" - - if force_reload: - self.load_data() - else: - psql_exists = self.check_psql() - if not psql_exists: + if len(non_none_inputs) > 0: + if self.esri_rest_urls is not None: + self.type = "esri" + elif self.carto_sql_queries is not None: + self.type = "carto" + elif self.gdf is not None: + self.type = "gdf" + + if force_reload: self.load_data() + else: + psql_exists = self.check_psql() + if not psql_exists: + self.load_data() + else: + print('Initialized FeatureLayer with no data.') def check_psql(self): try: @@ -145,7 +143,7 @@ def spatial_join(self, other_layer, how="left", predicate="intersects"): # If other_layer.gdf isn't a geodataframe, try to make it one if not isinstance(other_layer.gdf, gpd.GeoDataFrame): try: - other_layer.gdf = gpd.GeoDataFrame(other_layer.gdf, geometry="geometry") + other_layer.rebuild_gdf() except: print(other_layer.gdf) @@ -156,3 +154,6 @@ def spatial_join(self, other_layer, how="left", predicate="intersects"): self.gdf = gpd.sjoin(self.gdf, other_layer.gdf, how=how, predicate=predicate) self.gdf.drop(columns=["index_right"], inplace=True) self.gdf.drop_duplicates(inplace=True) + + def rebuild_gdf(self): + self.gdf = gpd.GeoDataFrame(self.gdf, geometry="geometry") \ No newline at end of file diff --git a/src/constants/services.py b/src/constants/services.py index c5ab1d6a..3c87684a 100644 --- a/src/constants/services.py +++ b/src/constants/services.py @@ -14,6 +14,10 @@ "https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/PHS_PhilaLandCare_Maintenance/FeatureServer/0/", ] +RCOS_LAYERS_TO_LOAD = [ + "https://services.arcgis.com/fLeGjb7u4uXqeF9q/ArcGIS/rest/services/Zoning_RCO/FeatureServer/0/" +] + one_year_ago = (datetime.datetime.now() - datetime.timedelta(days=365)).strftime( "%Y-%m-%d" ) @@ -22,3 +26,5 @@ COMPLAINTS_SQL_QUERY = f"SELECT address, service_request_id, subject, status, service_name, service_code, lat AS y, lon AS x FROM public_cases_fc WHERE requested_datetime >= '{one_year_ago}' AND lat IS NOT NULL;" VIOLATIONS_SQL_QUERY = f"SELECT parcel_id_num, casenumber, casecreateddate, casetype, casestatus, violationnumber, violationcodetitle, violationstatus, opa_account_num, address, opa_owner, geocode_x AS x, geocode_y AS y FROM violations WHERE violationdate >= '{one_year_ago}' AND geocode_x IS NOT NULL;" + +GUNCRIME_SQL_QUERY = f"SELECT text_general_code, dispatch_date, point_x AS x, point_y AS y FROM incidents_part1_part2 WHERE dispatch_date_time >= '{one_year_ago}' AND text_general_code IN ('Aggravated Assault Firearm', 'Robbery Firearm') AND point_x IS NOT NULL;" diff --git a/src/data_utils/gun_crimes.py b/src/data_utils/gun_crimes.py new file mode 100644 index 00000000..4fb2ffcc --- /dev/null +++ b/src/data_utils/gun_crimes.py @@ -0,0 +1,80 @@ +from constants.services import GUNCRIME_SQL_QUERY +from config.config import USE_CRS +from classes.featurelayer import FeatureLayer +import numpy as np +import matplotlib.pyplot as plt +from awkde.awkde import GaussianKDE +import rasterio +from rasterio.transform import Affine +import mapclassify + +def gun_crimes(primary_featurelayer): + # Initialize gun_crimes object + gun_crimes = FeatureLayer(name="Gun Crimes", carto_sql_queries=GUNCRIME_SQL_QUERY) + + # Extract x, y coordinates from geometry + x = np.array([]) + y = np.array([]) + + for geom in gun_crimes.gdf.geometry: + coords = np.array(geom.xy) + x = np.concatenate([x, coords[0]]) + y = np.concatenate([y, coords[1]]) + + # Prepare data for KDE + X = np.array(list(zip(x, y))) + + # Generate grid for plotting + x_grid, y_grid = np.linspace(x.min(), x.max(), 1000), np.linspace(y.min(), y.max(), 1000) + xx, yy = np.meshgrid(x_grid, y_grid) + grid_points = np.array([xx.ravel(), yy.ravel()]).T + + # Compute adaptive KDE values + kde = GaussianKDE(glob_bw="silverman", alpha=0.999, diag_cov=True) + kde.fit(X) + + z = kde.predict(grid_points) + zz = z.reshape(xx.shape) + + # Calculate resolutions and min values + x_res = (x.max() - x.min()) / (len(x_grid) - 1) + y_res = (y.max() - y.min()) / (len(y_grid) - 1) + min_x, min_y = x.min(), y.min() + + # Save the plot in tmp folder + plt.pcolormesh(xx, yy, zz) + plt.scatter(x, y, c='red', s=0.005) + plt.colorbar() + plt.savefig('tmp/kde.png') + + + # Define the affine transform + transform = Affine.translation(min_x, min_y) * Affine.scale(x_res, y_res) + + # Export as raster + with rasterio.open('tmp/output.tif', 'w', driver='GTiff', + height=zz.shape[0], + width=zz.shape[1], + count=1, dtype=zz.dtype, + crs=USE_CRS, + transform=transform) as dst: + dst.write(zz, 1) + + primary_featurelayer.gdf['centroid'] = primary_featurelayer.gdf.geometry.centroid + + coord_list = [(x,y) for x,y in zip(primary_featurelayer.gdf['centroid'].x, primary_featurelayer.gdf['centroid'].y)] + + primary_featurelayer.gdf = primary_featurelayer.gdf.drop(columns=['centroid']) + + src = rasterio.open('tmp/output.tif') + sampled_values = [x[0] for x in src.sample(coord_list)] + + primary_featurelayer.gdf['guncrime_density'] = sampled_values + + guncrime_classifier = mapclassify.Percentiles(primary_featurelayer.gdf['guncrime_density'], pct=[50, 75, 90, 95, 99, 100]) + primary_featurelayer.gdf['guncrime_density'] = primary_featurelayer.gdf['guncrime_density'].apply(guncrime_classifier) + primary_featurelayer.gdf['guncrime_density'] = primary_featurelayer.gdf['guncrime_density'].astype(float) + + primary_featurelayer.gdf['guncrime_density'] = primary_featurelayer.gdf['guncrime_density'].replace([0, 1, 2, 3, 4, 5], ['Bottom 50%', 'Top 50%', 'Top 25%', 'Top 10%', 'Top 5%', 'Top 1%']) + + return primary_featurelayer diff --git a/src/data_utils/l_and_i.py b/src/data_utils/l_and_i.py index 16389ab3..ca3c8bce 100644 --- a/src/data_utils/l_and_i.py +++ b/src/data_utils/l_and_i.py @@ -1,6 +1,5 @@ from classes.featurelayer import FeatureLayer from constants.services import COMPLAINTS_SQL_QUERY, VIOLATIONS_SQL_QUERY -import geopandas as gpd def l_and_i(primary_featurelayer): @@ -20,9 +19,7 @@ def l_and_i(primary_featurelayer): .apply(lambda x: "; ".join([val for val in x if val is not None])) .reset_index() ) - l_and_i_complaints.gdf = gpd.GeoDataFrame( - l_and_i_complaints.gdf, geometry="geometry" - ) + l_and_i_complaints.rebuild_gdf() # collapse complaints_gdf by address and concatenate the violationcodetitle values into a list with a semicolon separator # l_and_i_complaints.gdf = l_and_i_complaints.gdf.groupby('address')['service_name'].apply(lambda x: '; '.join([val for val in x if val is not None])).reset_index() @@ -80,9 +77,7 @@ def l_and_i(primary_featurelayer): .apply(lambda x: "; ".join([val for val in x if val is not None])) .reset_index() ) - l_and_i_complaints.gdf = gpd.GeoDataFrame( - l_and_i_complaints.gdf, geometry="geometry" - ) + l_and_i_complaints.rebuild_gdf() # rename the column to 'li_violations' l_and_i_violations.gdf.rename( diff --git a/src/data_utils/rco_geoms.py b/src/data_utils/rco_geoms.py new file mode 100644 index 00000000..f25c8a7a --- /dev/null +++ b/src/data_utils/rco_geoms.py @@ -0,0 +1,18 @@ +from classes.featurelayer import FeatureLayer +from constants.services import RCOS_LAYERS_TO_LOAD + + +def rco_geoms(primary_featurelayer): + rco_geoms = FeatureLayer( + name="RCOs", esri_rest_urls=RCOS_LAYERS_TO_LOAD + ) + + rco_aggregate_cols = ['ORGANIZATION_NAME', 'ORGANIZATION_ADDRESS', 'PRIMARY_EMAIL', 'PRIMARY_PHONE'] + + rco_geoms.gdf["rco_info"] = rco_geoms.gdf[rco_aggregate_cols].apply(lambda x: "; ".join(map(str, x)), axis=1) + "; " + rco_geoms.gdf['geometry'].apply(str) + rco_geoms.gdf = rco_geoms.gdf[['rco_info', 'geometry']] + rco_geoms.rebuild_gdf() + + primary_featurelayer.spatial_join(rco_geoms) + + return primary_featurelayer diff --git a/src/data_utils/tree_canopy.py b/src/data_utils/tree_canopy.py new file mode 100644 index 00000000..e6b451e7 --- /dev/null +++ b/src/data_utils/tree_canopy.py @@ -0,0 +1,30 @@ +import requests +import io +import zipfile +import geopandas as gpd +from classes.featurelayer import FeatureLayer +from config.config import USE_CRS + +def tree_canopy(primary_featurelayer): + tree_url = 'https://national-tes-data-share.s3.amazonaws.com/national_tes_share/pa.zip.zip' + + tree_response = requests.get(tree_url) + + with io.BytesIO(tree_response.content) as f: + with zipfile.ZipFile(f, 'r') as zip_ref: + zip_ref.extractall("tmp/") + + pa_trees = gpd.read_file("tmp/pa.shp") + pa_trees = pa_trees.to_crs(USE_CRS) + phl_trees = pa_trees[pa_trees['county'] == 'Philadelphia County'] + phl_trees = phl_trees[['tc_gap', 'geometry']] + + phl_trees.rename(columns={'tc_gap': 'tree_canopy_gap'}, inplace=True) + + + tree_canopy = FeatureLayer('Tree Canopy') + tree_canopy.gdf = phl_trees + + primary_featurelayer.spatial_join(tree_canopy) + + return primary_featurelayer diff --git a/src/script.py b/src/script.py index 15a43794..ae879cde 100644 --- a/src/script.py +++ b/src/script.py @@ -4,6 +4,9 @@ from data_utils.city_owned_properties import city_owned_properties from data_utils.phs_properties import phs_properties from data_utils.l_and_i import l_and_i +from data_utils.rco_geoms import rco_geoms +from data_utils.tree_canopy import tree_canopy +from data_utils.gun_crimes import gun_crimes """ @@ -33,10 +36,27 @@ """ vacant_properties = l_and_i(vacant_properties) +""" +Merge RCOs +""" +vacant_properties = rco_geoms(vacant_properties) + + +""" +Load Tree Canopy Data +""" +vacant_properties = tree_canopy(vacant_properties) + + +""" +Gun Crime Data +""" +vacant_properties = gun_crimes(vacant_properties) + + + +# Clean up vacant_properties.gdf.to_postgis( "vacant_properties_end", conn, if_exists="replace", index=False ) - -# Clean up - conn.close() From 68adcd01fe1ec052e14570fe949e42e14d327fb1 Mon Sep 17 00:00:00 2001 From: Brandon Cohen Date: Sun, 27 Aug 2023 20:21:59 -0400 Subject: [PATCH 6/7] formatting --- src/classes/featurelayer.py | 4 +-- src/data_utils/gun_crimes.py | 68 +++++++++++++++++++++++------------ src/data_utils/rco_geoms.py | 21 +++++++---- src/data_utils/tree_canopy.py | 20 ++++++----- src/script.py | 1 - 5 files changed, 74 insertions(+), 40 deletions(-) diff --git a/src/classes/featurelayer.py b/src/classes/featurelayer.py index fb87d6d9..7a2dbbf2 100644 --- a/src/classes/featurelayer.py +++ b/src/classes/featurelayer.py @@ -54,7 +54,7 @@ def __init__( if not psql_exists: self.load_data() else: - print('Initialized FeatureLayer with no data.') + print("Initialized FeatureLayer with no data.") def check_psql(self): try: @@ -156,4 +156,4 @@ def spatial_join(self, other_layer, how="left", predicate="intersects"): self.gdf.drop_duplicates(inplace=True) def rebuild_gdf(self): - self.gdf = gpd.GeoDataFrame(self.gdf, geometry="geometry") \ No newline at end of file + self.gdf = gpd.GeoDataFrame(self.gdf, geometry="geometry") diff --git a/src/data_utils/gun_crimes.py b/src/data_utils/gun_crimes.py index 4fb2ffcc..62e816ab 100644 --- a/src/data_utils/gun_crimes.py +++ b/src/data_utils/gun_crimes.py @@ -8,6 +8,7 @@ from rasterio.transform import Affine import mapclassify + def gun_crimes(primary_featurelayer): # Initialize gun_crimes object gun_crimes = FeatureLayer(name="Gun Crimes", carto_sql_queries=GUNCRIME_SQL_QUERY) @@ -25,7 +26,9 @@ def gun_crimes(primary_featurelayer): X = np.array(list(zip(x, y))) # Generate grid for plotting - x_grid, y_grid = np.linspace(x.min(), x.max(), 1000), np.linspace(y.min(), y.max(), 1000) + x_grid, y_grid = np.linspace(x.min(), x.max(), 1000), np.linspace( + y.min(), y.max(), 1000 + ) xx, yy = np.meshgrid(x_grid, y_grid) grid_points = np.array([xx.ravel(), yy.ravel()]).T @@ -43,38 +46,59 @@ def gun_crimes(primary_featurelayer): # Save the plot in tmp folder plt.pcolormesh(xx, yy, zz) - plt.scatter(x, y, c='red', s=0.005) + plt.scatter(x, y, c="red", s=0.005) plt.colorbar() - plt.savefig('tmp/kde.png') - + plt.savefig("tmp/kde.png") # Define the affine transform transform = Affine.translation(min_x, min_y) * Affine.scale(x_res, y_res) # Export as raster - with rasterio.open('tmp/output.tif', 'w', driver='GTiff', - height=zz.shape[0], - width=zz.shape[1], - count=1, dtype=zz.dtype, - crs=USE_CRS, - transform=transform) as dst: + with rasterio.open( + "tmp/output.tif", + "w", + driver="GTiff", + height=zz.shape[0], + width=zz.shape[1], + count=1, + dtype=zz.dtype, + crs=USE_CRS, + transform=transform, + ) as dst: dst.write(zz, 1) - - primary_featurelayer.gdf['centroid'] = primary_featurelayer.gdf.geometry.centroid - - coord_list = [(x,y) for x,y in zip(primary_featurelayer.gdf['centroid'].x, primary_featurelayer.gdf['centroid'].y)] - primary_featurelayer.gdf = primary_featurelayer.gdf.drop(columns=['centroid']) + primary_featurelayer.gdf["centroid"] = primary_featurelayer.gdf.geometry.centroid - src = rasterio.open('tmp/output.tif') - sampled_values = [x[0] for x in src.sample(coord_list)] + coord_list = [ + (x, y) + for x, y in zip( + primary_featurelayer.gdf["centroid"].x, + primary_featurelayer.gdf["centroid"].y, + ) + ] - primary_featurelayer.gdf['guncrime_density'] = sampled_values + primary_featurelayer.gdf = primary_featurelayer.gdf.drop(columns=["centroid"]) - guncrime_classifier = mapclassify.Percentiles(primary_featurelayer.gdf['guncrime_density'], pct=[50, 75, 90, 95, 99, 100]) - primary_featurelayer.gdf['guncrime_density'] = primary_featurelayer.gdf['guncrime_density'].apply(guncrime_classifier) - primary_featurelayer.gdf['guncrime_density'] = primary_featurelayer.gdf['guncrime_density'].astype(float) + src = rasterio.open("tmp/output.tif") + sampled_values = [x[0] for x in src.sample(coord_list)] - primary_featurelayer.gdf['guncrime_density'] = primary_featurelayer.gdf['guncrime_density'].replace([0, 1, 2, 3, 4, 5], ['Bottom 50%', 'Top 50%', 'Top 25%', 'Top 10%', 'Top 5%', 'Top 1%']) + primary_featurelayer.gdf["guncrime_density"] = sampled_values + + guncrime_classifier = mapclassify.Percentiles( + primary_featurelayer.gdf["guncrime_density"], pct=[50, 75, 90, 95, 99, 100] + ) + primary_featurelayer.gdf["guncrime_density"] = primary_featurelayer.gdf[ + "guncrime_density" + ].apply(guncrime_classifier) + primary_featurelayer.gdf["guncrime_density"] = primary_featurelayer.gdf[ + "guncrime_density" + ].astype(float) + + primary_featurelayer.gdf["guncrime_density"] = primary_featurelayer.gdf[ + "guncrime_density" + ].replace( + [0, 1, 2, 3, 4, 5], + ["Bottom 50%", "Top 50%", "Top 25%", "Top 10%", "Top 5%", "Top 1%"], + ) return primary_featurelayer diff --git a/src/data_utils/rco_geoms.py b/src/data_utils/rco_geoms.py index f25c8a7a..a508358e 100644 --- a/src/data_utils/rco_geoms.py +++ b/src/data_utils/rco_geoms.py @@ -3,14 +3,23 @@ def rco_geoms(primary_featurelayer): - rco_geoms = FeatureLayer( - name="RCOs", esri_rest_urls=RCOS_LAYERS_TO_LOAD - ) + rco_geoms = FeatureLayer(name="RCOs", esri_rest_urls=RCOS_LAYERS_TO_LOAD) - rco_aggregate_cols = ['ORGANIZATION_NAME', 'ORGANIZATION_ADDRESS', 'PRIMARY_EMAIL', 'PRIMARY_PHONE'] + rco_aggregate_cols = [ + "ORGANIZATION_NAME", + "ORGANIZATION_ADDRESS", + "PRIMARY_EMAIL", + "PRIMARY_PHONE", + ] - rco_geoms.gdf["rco_info"] = rco_geoms.gdf[rco_aggregate_cols].apply(lambda x: "; ".join(map(str, x)), axis=1) + "; " + rco_geoms.gdf['geometry'].apply(str) - rco_geoms.gdf = rco_geoms.gdf[['rco_info', 'geometry']] + rco_geoms.gdf["rco_info"] = ( + rco_geoms.gdf[rco_aggregate_cols].apply( + lambda x: "; ".join(map(str, x)), axis=1 + ) + + "; " + + rco_geoms.gdf["geometry"].apply(str) + ) + rco_geoms.gdf = rco_geoms.gdf[["rco_info", "geometry"]] rco_geoms.rebuild_gdf() primary_featurelayer.spatial_join(rco_geoms) diff --git a/src/data_utils/tree_canopy.py b/src/data_utils/tree_canopy.py index e6b451e7..bc133893 100644 --- a/src/data_utils/tree_canopy.py +++ b/src/data_utils/tree_canopy.py @@ -5,26 +5,28 @@ from classes.featurelayer import FeatureLayer from config.config import USE_CRS + def tree_canopy(primary_featurelayer): - tree_url = 'https://national-tes-data-share.s3.amazonaws.com/national_tes_share/pa.zip.zip' + tree_url = ( + "https://national-tes-data-share.s3.amazonaws.com/national_tes_share/pa.zip.zip" + ) tree_response = requests.get(tree_url) with io.BytesIO(tree_response.content) as f: - with zipfile.ZipFile(f, 'r') as zip_ref: + with zipfile.ZipFile(f, "r") as zip_ref: zip_ref.extractall("tmp/") - + pa_trees = gpd.read_file("tmp/pa.shp") pa_trees = pa_trees.to_crs(USE_CRS) - phl_trees = pa_trees[pa_trees['county'] == 'Philadelphia County'] - phl_trees = phl_trees[['tc_gap', 'geometry']] + phl_trees = pa_trees[pa_trees["county"] == "Philadelphia County"] + phl_trees = phl_trees[["tc_gap", "geometry"]] - phl_trees.rename(columns={'tc_gap': 'tree_canopy_gap'}, inplace=True) + phl_trees.rename(columns={"tc_gap": "tree_canopy_gap"}, inplace=True) - - tree_canopy = FeatureLayer('Tree Canopy') + tree_canopy = FeatureLayer("Tree Canopy") tree_canopy.gdf = phl_trees - + primary_featurelayer.spatial_join(tree_canopy) return primary_featurelayer diff --git a/src/script.py b/src/script.py index ae879cde..e1608111 100644 --- a/src/script.py +++ b/src/script.py @@ -54,7 +54,6 @@ vacant_properties = gun_crimes(vacant_properties) - # Clean up vacant_properties.gdf.to_postgis( "vacant_properties_end", conn, if_exists="replace", index=False From 3da0b7b623df9d317b048c3f44ecfa42c4cee35b Mon Sep 17 00:00:00 2001 From: Brandon Cohen Date: Sun, 27 Aug 2023 21:15:35 -0400 Subject: [PATCH 7/7] update gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 05e29933..575eaeda 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /notebooks/.ipynb_checkpoints/ .idea -src/tmp \ No newline at end of file +src/tmp +.DS_Store \ No newline at end of file