diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 54d60194d..000000000 --- a/.flake8 +++ /dev/null @@ -1,4 +0,0 @@ -[flake8] -max-line-length = 85 -ignore = E203, E241, E701, W503 -exclude = flycheck* \ No newline at end of file diff --git a/.github/workflows/build_docs.yaml b/.github/workflows/build_docs.yaml new file mode 100644 index 000000000..bec844c9f --- /dev/null +++ b/.github/workflows/build_docs.yaml @@ -0,0 +1,31 @@ +name: Build and deploy documentation +on: + push: + branches: + - master +jobs: + build-docs: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ['3.10'] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y pandoc + python -m pip install -e .[development] + - name: Build docs + run: cd doc && make html + - name: Deploy to GitHub Pages + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./doc/build/html diff --git a/.github/workflows/docs_check.yaml b/.github/workflows/docs_check.yaml new file mode 100644 index 000000000..9e150908a --- /dev/null +++ b/.github/workflows/docs_check.yaml @@ -0,0 +1,32 @@ +name: Check for Sphinx Warnings + +on: + pull_request: + paths: + - "doc/**" + - "**/*.rst" + - ".github/workflows/docs_check.yaml" + - "setup.py" + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Check out the repository + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y pandoc + python -m pip install -e .[development] + + - name: Check for Sphinx warnings + run: | + sphinx-build -M html ./doc/source ./doc/_build --fail-on-warning diff --git a/.github/workflows/format_check.yml b/.github/workflows/format_check.yml new file mode 100644 index 000000000..294f1e458 --- /dev/null +++ b/.github/workflows/format_check.yml @@ -0,0 +1,11 @@ +name: Ruff format +on: [push, pull_request] +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: chartboost/ruff-action@v1 + with: + args: 'format --check' + version: 0.7.0 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..2546c0edb --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,10 @@ +name: Ruff check +on: [push, pull_request] +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: chartboost/ruff-action@v1 + with: + version: 0.7.0 diff --git a/.github/workflows/spell_check.yml b/.github/workflows/spell_check.yml new file mode 100644 index 000000000..c894573a1 --- /dev/null +++ b/.github/workflows/spell_check.yml @@ -0,0 +1,15 @@ +name: Spell Check + +on: [push, pull_request] + +jobs: + spell-check: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Run codespell + uses: codespell-project/actions-codespell@v2 + diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 25bd3f43a..fbff69080 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: ['3.8', '3.9', '3.10'] + python-version: ['3.9', '3.10', '3.11','3.12'] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/.gitignore b/.gitignore index 5b7a7e797..400bdc0de 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,7 @@ __pycache__ flycheck*.py /notes/ /.ropeproject/ + +/doc_src/ +/doc/build/ +/doc/source/api_reference/_autosummary diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..ff32007f3 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "pelicun/resources/DamageAndLossModelLibrary"] + path = pelicun/resources/DamageAndLossModelLibrary + url = https://github.com/NHERI-SimCenter/DamageAndLossModelLibrary diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index e38360dd6..000000000 --- a/.pylintrc +++ /dev/null @@ -1,571 +0,0 @@ -[MAIN] - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -init-hook='import sys; sys.path.append("."); sys.path.append("../"); sys.path.append("../../")' - -# Files or directories to be skipped. They should be base names, not -# paths. -ignore=flycheck_* - -# Add files or directories matching the regex patterns to the ignore-list. The -# regex matches against paths and can be in Posix or Windows format. -ignore-paths=rulesets - -# Files or directories matching the regex patterns are skipped. The regex -# matches against base names, not paths. -ignore-patterns=^\.# - -# Pickle collected data for later comparisons. -persistent=yes - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - pylint.extensions.check_elif, - pylint.extensions.bad_builtin, - pylint.extensions.for_any_all, - pylint.extensions.set_membership, - pylint.extensions.code_style, - pylint.extensions.overlapping_exceptions, - pylint.extensions.typing, - pylint.extensions.redefined_variable_type, - pylint.extensions.comparison_placement, - pylint.extensions.docparams - -# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the -# number of processors available to use. -jobs=0 - -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages. -suggestion-mode=yes - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-allow-list= - -# Minimum supported python version -py-version = 3.7.2 - -# Control the amount of potential inferred values when inferring a single -# object. This can help the performance when dealing with large functions or -# complex, nested conditions. -limit-inference-results=100 - -# Specify a score threshold under which the program will exit with error. -fail-under=10.0 - -# Return non-zero exit code if any of these messages/categories are detected, -# even if score is above --fail-under value. Syntax same as enable. Messages -# specified are enabled, while categories only check already-enabled messages. -fail-on= - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -# confidence= - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable= - use-symbolic-message-instead, - useless-suppression, - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then re-enable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" - -disable= - attribute-defined-outside-init, - invalid-name, - missing-param-doc, - missing-type-doc, - protected-access, - too-few-public-methods, - # handled by black - format, - # We anticipate #3512 where it will become optional - fixme, - -[REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages -reports=no - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables 'fatal', 'error', 'warning', 'refactor', 'convention' -# and 'info', which contain the number of messages in each category, as -# well as 'statement', which is the total number of statements analyzed. This -# score is used by the global evaluation report (RP0004). -evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - -# Activate the evaluation score. -score=yes - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - -# The type of string formatting that logging methods do. `old` means using % -# formatting, `new` is for `{}` formatting. -logging-format-style=old - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO,todo,debug - -# Regular expression of note tags to take in consideration. -#notes-rgx= - - -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=8 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=yes - -# Signatures are removed from the similarity computation -ignore-signatures=yes - - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_$|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid defining new builtins when possible. -additional-builtins= - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of names allowed to shadow builtins -allowed-redefined-builtins= - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore. -ignored-argument-names=_.* - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io - - -[FORMAT] - -# Maximum number of characters on a single line. -max-line-length=85 - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Maximum number of lines in a module -max-module-lines=2000 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - - -[BASIC] - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Good variable names regexes, separated by a comma. If names match any regex, -# they will always be accepted -good-names-rgxs= - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Bad variable names regexes, separated by a comma. If names match any regex, -# they will always be refused -bad-names-rgxs= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# Naming style matching correct function names. -function-naming-style=snake_case - -# Regular expression matching correct function names -function-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming style matching correct variable names. -variable-naming-style=snake_case - -# Regular expression matching correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming style matching correct constant names. -const-naming-style=UPPER_CASE - -# Regular expression matching correct constant names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Naming style matching correct attribute names. -attr-naming-style=snake_case - -# Regular expression matching correct attribute names -attr-rgx=[a-z_][a-z0-9_]{2,}$ - -# Naming style matching correct argument names. -argument-naming-style=snake_case - -# Regular expression matching correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming style matching correct class attribute names. -class-attribute-naming-style=any - -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming style matching correct class constant names. -class-const-naming-style=UPPER_CASE - -# Regular expression matching correct class constant names. Overrides class- -# const-naming-style. -#class-const-rgx= - -# Naming style matching correct inline iteration names. -inlinevar-naming-style=any - -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming style matching correct class names. -class-naming-style=PascalCase - -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - - -# Naming style matching correct module names. -module-naming-style=snake_case - -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - - -# Naming style matching correct method names. -method-naming-style=snake_case - -# Regular expression matching correct method names -method-rgx=[a-z_][a-z0-9_]{2,}$ - -# Regular expression matching correct type variable names -#typevar-rgx= - -# Regular expression which should only match function or class names that do -# not require a docstring. Use ^(?!__init__$)_ to also check __init__. -no-docstring-rgx=((^test_)|(^_.*(? - pelicun

@@ -12,8 +12,11 @@ Probabilistic Estimation of Losses, Injuries, and Community resilience Under Natural hazard events

+[![Latest Release](https://img.shields.io/github/v/release/NHERI-SimCenter/pelicun?color=blue&label=Latest%20Release)](https://github.com/NHERI-SimCenter/pelicun/releases/latest) ![Tests](https://github.com/NHERI-SimCenter/pelicun/actions/workflows/tests.yml/badge.svg) [![codecov](https://codecov.io/github/NHERI-SimCenter/pelicun/branch/master/graph/badge.svg?token=W79M5FGOCG)](https://codecov.io/github/NHERI-SimCenter/pelicun/tree/master) +[![Ruff](https://img.shields.io/badge/ruff-linted-blue)](https://img.shields.io/badge/ruff-linted-blue) +[![License](https://img.shields.io/badge/License-BSD%203--Clause-blue)](https://raw.githubusercontent.com/NHERI-SimCenter/pelicun/master/LICENSE) ## What is it? @@ -43,8 +46,6 @@ Detailed documentation of the available methods and their use is available at ht ## Installation -### For users - `pelicun` is available at the [Python Package Index (PyPI)](https://pypi.org/project/pelicun/). You can simply install it using `pip` as follows: ```shell @@ -59,206 +60,14 @@ pip install pelicun==2.6.0 Note that 2.6.0 is the last minor version before the v3.0 release. Other earlier versions can be found [here](https://pypi.org/project/pelicun/#history). -### For contributors - -Developers are expected to fork and clone this repository, and install their copy in development mode. -Using a virtual environment is highly recommended. - -```shell -# Clone the repository: -git clone https://github.com//pelicun -cd pelicun -# Switch to the appropriate branch, if needed. -# git checkout - -# Install pelicun: -# Note: don't forget to activate the corresponding environment. -python -m pip install -e .[development] - -``` - -Contributions are managed with pull requests. -It is required that contributed code is [PEP 8](https://peps.python.org/pep-0008/) compliant, does not introduce linter warnings and includes sufficient unit tests so as to avoid reducing the current coverage level. - -The following lines implement the aforementioned checks. -`flake8`, `pylint` and `pytest` can all be configured for use within an IDE. -```shell - -cd /pelicun -export PYTHONPATH=$PYTHONPATH:$(pwd) - -# Linting with flake8: -flake8 pelicun - -# Linting with pylint: -pylint pelicun - -# Type checking with mypy: -mypy pelicun --no-namespace-packages - -# Running the tests: -python -m pytest pelicun/tests --cov=pelicun --cov-report html -# Open `htmlcov/index.html`in a browser to see coverage results. - -``` - -Feel free to [open an issue](https://github.com/NHERI-SimCenter/pelicun/issues/new/choose) if you encounter problems setting up the provided development environment. +## Documentation and usage examples +The documentation for pelicun can be accessed [here](https://NHERI-SimCenter.github.io/pelicun/). +It includes information for users, instructions for developers and usage examples. ## Changelog -### Changes in v3.3 - -- Changes affecting backwards compatibility - - - **Remove "bldg" from repair consequence output filenames**: The increasing scope of Pelicun now covers simulations for transportation and water networks. Hence, labeling repair consequence outputs as if they were limited to buildings no longer seems appropriate. The `bldg` label was dropped from the following files: `DV_bldg_repair_sample`,`DV_bldg_repair_stats`,`DV_bldg_repair_grp`, `DV_bldg_repair_grp_stats`, `DV_bldg_repair_agg`, `DV_bldg_repair_agg_stats`. - -- Deprecation warnings - - - **Remove `Bldg` from repair settings label in DL configuration file**: Following the changes above, we dropped `Bldg` from `BldgRepair` when defining settings for repair consequence simulation in a configuration file. The previous version (i.e., `BldgRepair`) will keep working until the next major release, but we encourage everyone to adopt the new approach and simply use the `Repair` keyword there. - -- New features - - - **Location-specific damage processes**: This new feature is useful when you want damage to a component type to induce damage in another component type at the same location only. For example, damaged water pipes on a specific story can trigger damage in floor covering only on that specific story. Location-matching is performed automatically without you having to define component pairs for every location using the following syntax: `'1_CMP.A-LOC', {'DS1': 'CMP.B_DS1'}` , where DS1 of `CMP.A` at each location triggers DS1 of `CMP.B` at the same location. - - - **New `custom_model_dir` argument for `DL_calculation`**: This argument allows users to prepare custom damage and loss model files in a folder and pass the path to that folder to an auto-population script through `DL_calculation`. Within the auto-population script, they can reference only the name of the files in that folder. This provides portability for simulations that use custom models and auto population, such as some of the advanced regional simualtions in [SimCenter's R2D Tool](https://simcenter.designsafe-ci.org/research-tools/r2dtool/). - - - **Extend Hazus EQ auto population sripts to include water networks**: Automatically recognize water network assets and map them to archetypes from the Hazus Earthquake technical manual. - - - **Introduce `convert_units` function**: Provide streamlined unit conversion using the pre-defined library of units in Pelicun. Allows you to convert a variable from one unit to another using a single line of simple code, such as - `converted_height = pelicun.base.convert_units(raw_height, unit='m', to_unit='ft')` - While not as powerful as some of the Python packages dedicated to unit conversion (e.g., [Pint](https://pint.readthedocs.io/en/stable/)), we believe the convenience this function provides for commonly used units justifies its use in several cases. - -- Architectural and code updates - - - **Split `model.py` into subcomponents**: The `model.py` file was too large and its contents were easy to refactor into separate modules. Each model type has its own python file now and they are stored under the `model` folder. - - - **Split the `RandomVariable` class into specific classes**: It seems more straightforward to grow the list of supported random variables by having a specific class for each kind of RV. We split the existing large `RandomVariable` class in `uq.py` leveraging inheritance to minimize redundant code. - - - **Automatic code formatting**: Further improve consistency in coding style by using [black](https://black.readthedocs.io/en/stable/) to review and format the code when needed. - - - **Remove `bldg` from variable and class names**: Following the changes mentioned earlier, we dropped `bldg` from lables where the functionality is no longer limited to buildings. - - - **Introduce `calibrated` attribute for demand model**: This new attribute will allow users to check if a model has already been calibrated to the provided empirical data. - - - Several other minor improvements; see commit messages for details. - -- Dependencies - - - Ceiling raised for `pandas`, supporting version 2.0 and above up until 3.0. - -### Changes in v3.2 - -- Changes that might affect backwards compatibility: - - - Unit information is included in every output file. If you parse Pelicun outputs and did not anticipate a Unit entry, your parser might need an update. - - - Decision variable types in the repair consequence outputs are named using CamelCase rather than all capitals to be consistent with other parts of the codebase. For example, we use "Cost" instead of "COST". This might affect post-processing scripts. - - - For clarity, "ea" units were replaced with "unitless" where appropriate. There should be no practical difference between the calculations due to this change. Interstory drift ratio demand types are one example. - - - Weighted component block assignment is no longer supported. We recommend using more versatile multiple component definitions (see new feature below) to achieve the same effect. - - - Damage functions (i.e., assign quantity of damage as a function of demand) are no longer supported. We recommend using the new multilinear CDF feature to develop theoretically equivalent, but more efficient models. - -- New multilinear CDF Random Variable allows using the multilinear approximation of any CDF in the tool. - -- Capacity adjustment allows adjusting (scaling or shifting) default capacities (i.e., fragility curves) with factors specific to each Performance Group. - -- Support for multiple definitions of the same component at the same location-direction. This feature facilitates adding components with different block sizes to the same floor or defining multiple tenants on the same floor, each with their own set of components. - -- Support for cloning demands, that is, taking a provided demand dataset, creating a copy and considering it as another demand. For example, you can provide results of seismic response in the X direction and automatically prepare a copy of them to represent results in the Y direction. - -- Added a comprehensive suite of more than 140 unit tests that cover more than 93% of the codebase. Tests are automatically executed after every commit using GitHub Actions and coverage is monitored through `Codecov.io`. Badges at the top of the Readme show the status of tests and coverage. We hope this continuous integration facilitates editing and extending the existing codebase for interested members of the community. - -- Completed a review of the entire codebase using `flake8` and `pylint` to ensure PEP8 compliance. The corresponding changes yielded code that is easier to read and use. See guidance in Readme on linting and how to ensure newly added code is compliant. - -- Models for estimating Environmental Impact (i.e., embodied carbon and energy) of earthquake damage as per FEMA P-58 are included in the DL Model Library and available in this release. - -- "ListAllDamageStates" option allows you to print a comprehensive list of all possible damage states for all components in the columns of the DMG output file. This can make parsing the output easier but increases file size. By default, this option is turned off and only damage states that affect at least one block are printed. - -- Damage and Loss Model Library - - - A collection of parameters and metadata for damage and loss models for performance based engineering. The library is available and updated regularly in the DB_DamageAndLoss GitHub Repository. - - - This and future releases of Pelicun have the latest version of the library at the time of their release bundled with them. - -- DL_calculation tool - - - Support for combination of built-in and user-defined databases for damage and loss models. - - - Results are now also provided in standard SimCenter `JSON` format besides the existing `CSV` tables. You can specify the preferred format in the configuration file under Output/Format. The default file format is still CSV. - - - Support running calculations for only a subset of available consequence types. - -- Several error and warning messages added to provide more meaningful information in the log file when something goes wrong in a simulation. - -- Update dependencies to more recent versions. - -- The online documentation is significantly out of date. While we are working on an update, we recommend using the documentation of the [DL panel in SimCenter's PBE Tool](https://nheri-simcenter.github.io/PBE-Documentation/common/user_manual/usage/desktop/PBE/Pelicun.html) as a resource. - -### Changes in v3.1 - -- Calculation settings are now assessment-specific. This allows you to use more than one assessments in an interactive calculation and each will have its own set of options, including log files. - -- The uq module was decoupled from the others to enable standalone uq calculations that work without having an active assessment. - -- A completely redesigned DL_calculation.py script that provides decoupled demand, damage, and loss assessment and more flexibility when setting up each of those when pelicun is used with a configuration file in a larger workflow. - -- Two new examples that use the DL_calculation.py script and a json configuration file were added to the example folder. - -- A new example that demonstrates a detailed interactive calculation in a Jupyter notebook was added to the following DesignSafe project: https://www.designsafe-ci.org/data/browser/public/designsafe.storage.published/PRJ-3411v5 This project will be extended with additional examples in the future. - -- Unit conversion factors moved to an external file (settings/default_units) to make it easier to add new units to the list. This also allows redefining the internal units through a complete replacement of the factors. The internal units continue to follow the SI system. - -- Substantial improvements in coding style using flake8 and pylint to monitor and help enforce PEP8. - -- Several performance improvements made calculations more efficient, especially for large problems, such as regional assessments or tall buildings investigated using the FEMA P-58 methodology. - -- Several bugfixes and a large number of minor changes that make the engine more robust and easier to use. - -- Update recommended Python version to 3.10 and other dependencies to more recent versions. - -### Changes in v3.0 - -- The architecture was redesigned to better support interactive calculation and provide a low-level integration across all supported methods. This is the first release with the new architecture. Frequent updates are planned to provide additional examples, tests, and bugfixes in the next few months. - -- New `assessment` module introduced to replace `control` module: - - Provides a high-level access to models and their methods - - Integrates all types of assessments into a uniform approach - - Most of the methods from the earlier `control` module were moved to the `model` module - -- Decoupled demand, damage, and loss calculations: - - Fragility functions and consequence functions are stored in separate files. Added new methods to the `db` module to prepare the corresponding data files and re-generated such data for FEMA P58 and Hazus earthquake assessments. Hazus hurricane data will be added in a future release. - - Decoupling removed a large amount of redundant data from supporting databases and made the use of HDF and json files for such data unnecessary. All data are stored in easy-to-read csv files. - - Assessment workflows can include all three steps (i.e., demand, damage, and loss) or only one or two steps. For example, damage estimates from one analysis can drive loss calculations in another one. - -- Integrated damage and loss calculation across all methods and components: - - This includes phenomena such as collapse, including various collapse modes, and irreparable damage. - - Cascading damages and other interdependencies between various components can be introduced using a damage process file. - - Losses can be driven by damages or demands. The former supports the conventional damage->consequence function approach, while the latter supports the use of vulnerability functions. These can be combined within the same analysis, if needed. - - The same loss component can be driven by multiple types of damages. For example, replacement can be triggered by either collapse or irreparable damage. - -- Introduced *Options* in the configuration file and in the `base` module: - - These options handle settings that concern pelicun behavior; - - general preferences that might affect multiple assessment models; - - and settings that users would not want to change frequently. - - Default settings are provided in a `default_config.json` file. These can be overridden by providing any of the prescribed keys with a user-defined value assigned to them in the configuration file for an analysis. - -- Introduced consistent handling of units. Each csv table has a standard column to describe units of the data in it. If the standard column is missing, the table is assumed to use SI units. - -- Introduced consistent handling of pandas MultiIndex objects in headers and indexes. When tabular data is stored in csv files, MultiIndex objects are converted to simple indexes by concatenating the strings at each level and separating them with a `-`. This facilitates post-processing csv files in pandas without impeding post-processing those files in non-Python environments. - -- Updated the DL_calculation script to support the new architecture. Currently, only the config file input is used. Other arguments were kept in the script for backwards compatibility; future updates will remove some of those arguments and introduce new ones. - -- The log files were redesigned to provide more legible and easy-to-read information about the assessment. - -### Changes in v2.6 - -- Support EDPs with more than 3 characters and/or a variable in their name. For example, SA_1.0 or SA_T1 -- Support fitting normal distribution to raw EDP data (lognormal was already available) -- Extract key settings to base.py to make them more accessible for users. -- Minor bugfixes mostly related to hurricane storm surge assessment +The release notes are available in the [online documentation](https://nheri-simcenter.github.io/pelicun/release_notes/index.html) ## License diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 000000000..d0c3cbf10 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/doc/make.bat b/doc/make.bat new file mode 100644 index 000000000..dc1312ab0 --- /dev/null +++ b/doc/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/doc/source/_extensions/latest_citation.py b/doc/source/_extensions/latest_citation.py new file mode 100644 index 000000000..f2112bb8e --- /dev/null +++ b/doc/source/_extensions/latest_citation.py @@ -0,0 +1,56 @@ +# noqa: INP001, CPY001, D100 +import requests +from docutils import nodes +from docutils.parsers.rst import Directive + + +class LatestCitationDirective(Directive): # noqa: D101 + def run(self): # noqa: ANN201, D102 + citation_text, bibtex_text = self.get_latest_zenodo_citation() + + # Create nodes for the standard citation and BibTeX + citation_node = nodes.paragraph(text=citation_text) + bibtex_node = nodes.literal_block(text=bibtex_text, language='bibtex') + + return [citation_node, bibtex_node] + + def get_latest_zenodo_citation(self): # noqa: PLR6301, ANN201, D102 + url = 'https://zenodo.org/api/records/?q=conceptdoi:10.5281/zenodo.2558557&sort=mostrecent' + try: + response = requests.get(url) # noqa: S113 + except requests.exceptions.ConnectionError: + return '(No Connection)', '' + data = response.json() + latest_record = data['hits']['hits'][0] + authors = [ + author['name'] for author in latest_record['metadata']['creators'] + ] + combine_chars = [', '] * (len(authors) - 2) + [', and '] + author_str = authors[0] + for author, combine_char in zip(authors[1::], combine_chars): + author_str += combine_char + author + title = latest_record['metadata']['title'].split(': ')[0] + version = latest_record['metadata']['version'] + doi = latest_record['metadata']['doi'] + year = latest_record['metadata']['publication_date'][:4] + month = latest_record['metadata']['publication_date'][5:7] # noqa: F841 + publisher = 'Zenodo' + + # Standard citation + citation_text = f'{author_str} ({year}) {title}. DOI:{doi}' + + # BibTeX citation + bibtex_text = f"""@software{{{author_str.replace(" ", "_").replace(",", "").replace("_and_", "_").lower()}_{year}_{doi.split('.')[-1]}, + author = {{{" and ".join(authors)}}}, + title = {{{title}}}, + year = {year}, + publisher = {{{publisher}}}, + version = {{{version}}}, + doi = {{{doi}}}, +}}""" + + return citation_text, bibtex_text + + +def setup(app): # noqa: ANN201, D103, ANN001 + app.add_directive('latest-citation', LatestCitationDirective) diff --git a/doc/source/_static/css/custom.css b/doc/source/_static/css/custom.css new file mode 100644 index 000000000..17f191d76 --- /dev/null +++ b/doc/source/_static/css/custom.css @@ -0,0 +1,33 @@ +wy-nav-content { + max-width: none; +} + + +.math { + text-align: left; +} + +.eqno { + float: right; +} + + +#div.wy-side-scroll{ +# background:#cb463f; +#} + +div.wy-menu.wy-menu-vertical > .caption { + color: #cb463f; +} + +# LIGHT BLUE background:#0099ff +# BLUE: background:#0B619C +# ADAM RED: background:#cb463f; + +span.caption.caption-text{ + color: #000000; +} + +td{ + white-space: normal !important; +} diff --git a/doc/source/_static/front_page/api-svgrepo-com.svg b/doc/source/_static/front_page/api-svgrepo-com.svg new file mode 100644 index 000000000..4323ed226 --- /dev/null +++ b/doc/source/_static/front_page/api-svgrepo-com.svg @@ -0,0 +1,150 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/front_page/book-svgrepo-com.svg b/doc/source/_static/front_page/book-svgrepo-com.svg new file mode 100644 index 000000000..7d709ce92 --- /dev/null +++ b/doc/source/_static/front_page/book-svgrepo-com.svg @@ -0,0 +1,45 @@ + + + + + + + + diff --git a/doc/source/_static/front_page/programmer-svgrepo-com.svg b/doc/source/_static/front_page/programmer-svgrepo-com.svg new file mode 100644 index 000000000..92540aed4 --- /dev/null +++ b/doc/source/_static/front_page/programmer-svgrepo-com.svg @@ -0,0 +1,85 @@ + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/front_page/right-arrow-svgrepo-com.svg b/doc/source/_static/front_page/right-arrow-svgrepo-com.svg new file mode 100644 index 000000000..a0e8e1150 --- /dev/null +++ b/doc/source/_static/front_page/right-arrow-svgrepo-com.svg @@ -0,0 +1,41 @@ + + + + + + diff --git a/doc/source/_static/hide_empty_pre.js b/doc/source/_static/hide_empty_pre.js new file mode 100644 index 000000000..e66f1dd43 --- /dev/null +++ b/doc/source/_static/hide_empty_pre.js @@ -0,0 +1,13 @@ +document.addEventListener("DOMContentLoaded", function() { + document.querySelectorAll('div.nboutput.docutils.container').forEach(function(div) { + // Our objective is to hide all `div` elements of which all + // children elements only contain whitespace. + // This remedies the nbsphinx issue where an extra newline was + // added to each line in the code block output. + let isEmpty = Array.from(div.children).every(child => !child.textContent.trim()); + + if (isEmpty) { + div.style.display = 'none'; + } + }); +}); diff --git a/doc/source/_static/pelicun-Logo-grey.png b/doc/source/_static/pelicun-Logo-grey.png new file mode 100644 index 000000000..eda5c6845 Binary files /dev/null and b/doc/source/_static/pelicun-Logo-grey.png differ diff --git a/doc/source/_static/pelicun-Logo-white.png b/doc/source/_static/pelicun-Logo-white.png new file mode 100644 index 000000000..a799a6557 Binary files /dev/null and b/doc/source/_static/pelicun-Logo-white.png differ diff --git a/doc/source/_static/pelicun-Logo.png b/doc/source/_static/pelicun-Logo.png new file mode 100644 index 000000000..991123331 Binary files /dev/null and b/doc/source/_static/pelicun-Logo.png differ diff --git a/doc/source/_templates/custom-class-template.rst b/doc/source/_templates/custom-class-template.rst new file mode 100644 index 000000000..b29757c52 --- /dev/null +++ b/doc/source/_templates/custom-class-template.rst @@ -0,0 +1,32 @@ +{{ fullname | escape | underline}} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :members: + :show-inheritance: + :inherited-members: + + {% block methods %} + .. automethod:: __init__ + + {% if methods %} + .. rubric:: {{ _('Methods') }} + + .. autosummary:: + {% for item in methods %} + ~{{ name }}.{{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Attributes') }} + + .. autosummary:: + {% for item in attributes %} + ~{{ name }}.{{ item }} + {%- endfor %} + {% endif %} + {% endblock %} diff --git a/doc/source/_templates/custom-module-template.rst b/doc/source/_templates/custom-module-template.rst new file mode 100644 index 000000000..dc5355649 --- /dev/null +++ b/doc/source/_templates/custom-module-template.rst @@ -0,0 +1,63 @@ +{{ fullname | escape | underline}} + +.. automodule:: {{ fullname }} + :members: + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Module Attributes') }} + + .. autosummary:: + {% for item in attributes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block functions %} + {% if functions %} + .. rubric:: {{ _('Functions') }} + + .. autosummary:: + {% for item in functions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block classes %} + {% if classes %} + .. rubric:: {{ _('Classes') }} + + .. autosummary:: + :template: custom-class-template.rst + {% for item in classes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block exceptions %} + {% if exceptions %} + .. rubric:: {{ _('Exceptions') }} + + .. autosummary:: + {% for item in exceptions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + +{% block modules %} +{% if modules %} +.. rubric:: Modules + +.. autosummary:: + :toctree: + :template: custom-module-template.rst + :recursive: +{% for item in modules %} + {{ item }} +{%- endfor %} +{% endif %} +{% endblock %} diff --git a/doc/source/about/LICENSE b/doc/source/about/LICENSE new file mode 100644 index 000000000..0c0b603fa --- /dev/null +++ b/doc/source/about/LICENSE @@ -0,0 +1,32 @@ +This source code is licensed under a BSD 3-Clause License. + +Copyright (c) 2018 Leland Stanford Junior University +Copyright (c) 2018 The Regents of the University of California + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/doc/source/about/acknowledgments.rst b/doc/source/about/acknowledgments.rst new file mode 100644 index 000000000..20cf944c8 --- /dev/null +++ b/doc/source/about/acknowledgments.rst @@ -0,0 +1,39 @@ +.. _acknowledgments: + +*************** +Acknowledgments +*************** + +--------------------------- +National Science Foundation +--------------------------- + +This material is based upon work supported by the National Science Foundation under Grant No. 1612843. Any opinions, findings, and conclusions or recommendations expressed in this material are those of the author(s) and do not necessarily reflect the views of the National Science Foundation. + +------------ +Contributors +------------ + +The developers are grateful to the researchers and experts listed below for their insights and suggestions that contributed to the development of pelicun. + +**Jack W. Baker** | Stanford University + +**Tracy Becker** | University of California Berkeley + +**Gregory G. Deierlein** | Stanford University + +**Anne Kiremidjian** | Stanford University + +**Pouria Kourehpaz** | University of British Columbia + +**Carlos Molina Hutt** | University of British Columbia + +**Vesna Terzic** | California State University Long Beach + +**Paola Vargas** | University of Michigan + +**David P. Welch** | Stanford University + +**Major Zeng** | Stanford University + +**Joanna J. Zou** | Stanford University diff --git a/doc/source/about/cite.rst b/doc/source/about/cite.rst new file mode 100644 index 000000000..9301d3a4c --- /dev/null +++ b/doc/source/about/cite.rst @@ -0,0 +1,36 @@ +.. _cite: + +Citing pelicun +-------------- + +When referencing pelicun in your research or publications, please use the following citation. +Proper citation is crucial for acknowledging the efforts of the development team and ensuring the reproducibility of your work. + +.. card:: Latest pelicun citation + :link: https://zenodo.org/doi/10.5281/zenodo.2558557 + + .. latest-citation:: + + +Logo +---- + +.. image:: ../_static/pelicun-Logo-white.png + :alt: NHERI-SimCenter pelicun logo + :align: center + +The pelicun logo is a trademark of NHERI-SimCenter and is protected under applicable trademark laws. +You are permitted to use the pelicun logo under the following conditions: + +1. **Non-Commercial Use**: The logo may be used for non-commercial purposes, including academic publications, presentations, and educational materials, provided that such use is directly related to the pelicun software. + +2. **Integrity of the Logo**: The logo must not be altered, modified, or distorted in any way. + This includes changes to the logo's proportions, colors, and text, except for resizing that maintains the original aspect ratio. + +3. **Attribution**: Any use of the logo must be accompanied by an attribution to "NHERI-SimCenter" as the owner of the pelicun logo. + +4. **Prohibited Uses**: The logo must not be used in any manner that suggests endorsement or sponsorship by NHERI-SimCenter of any third-party products, services, or organizations, unless explicit permission has been granted. + +5. **Legal Compliance**: The use of the pelicun logo must comply with all applicable laws and regulations, including those related to trademark and intellectual property rights. + +For any uses not covered by the above terms, or to seek permission for commercial use, please contact NHERI-SimCenter directly. diff --git a/doc/source/about/license.rst b/doc/source/about/license.rst new file mode 100644 index 000000000..9be2f6a7b --- /dev/null +++ b/doc/source/about/license.rst @@ -0,0 +1,9 @@ +.. _license: + +Copyright and license +--------------------- + +Pelicun is copyright "Leland Stanford Junior University and The Regents of the University of California," and is licensed under the following BSD license: + +.. literalinclude:: LICENSE + :language: none diff --git a/doc/source/api_reference/index.rst b/doc/source/api_reference/index.rst new file mode 100644 index 000000000..0d1953c1d --- /dev/null +++ b/doc/source/api_reference/index.rst @@ -0,0 +1,16 @@ +.. + Courtesy of James Leedham + https://stackoverflow.com/questions/2701998/automatically-document-all-modules-recursively-with-sphinx-autodoc + +.. _api_reference: + +============= +API Reference +============= + +.. autosummary:: + :toctree: _autosummary + :template: custom-module-template.rst + :recursive: + + pelicun diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 000000000..e208ef0f5 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,108 @@ +# noqa: INP001, CPY001 +"""Pelicun Sphinx configuration.""" + +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +from datetime import datetime +from pathlib import Path + +sys.path.insert(0, str(Path('./_extensions').resolve())) + +# -- Project information ----------------------------------------------------- +project = 'pelicun' +copyright = ( # noqa: A001 + f'{datetime.now().year}, Leland Stanford Junior ' # noqa: DTZ005 + f'University and The Regents of the University of California' +) +author = 'Adam Zsarnóczay' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'numpydoc', + 'sphinx_design', + 'nbsphinx', + 'sphinxcontrib.bibtex', + 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.mathjax', + 'sphinx.ext.viewcode', + 'sphinx.ext.githubpages', + 'sphinx.ext.autosummary', + 'sphinx.ext.intersphinx', + 'sphinx.ext.doctest', + # our own extension to get latest citation from zenodo. + 'latest_citation', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +html_css_files = ['css/custom.css'] +html_js_files = ['hide_empty_pre.js'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', '**/tests/*'] + +# Extension configuration + +autosummary_generate = True # Turn on sphinx.ext.autosummary + +intersphinx_mapping = { + 'python': ('https://docs.python.org/3/', None), + 'numpy': ('http://docs.scipy.org/doc/numpy/', None), + 'scipy': ('http://docs.scipy.org/doc/scipy/reference/', None), +} + +numpydoc_show_class_members = False # TODO(JVM): remove and extend docstrings + +nbsphinx_custom_formats = { + '.pct.py': ['jupytext.reads', {'fmt': 'py:percent'}], +} + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +html_logo = '_static/pelicun-Logo-grey.png' +html_theme_options = { + 'analytics_id': 'UA-158130480-7', + 'logo_only': True, + 'collapse_navigation': False, + 'prev_next_buttons_location': None, + 'navigation_depth': 2, + 'style_nav_header_background': '#F2F2F2', +} +html_show_sphinx = False # Removes "Built with Sphinx using a theme [...]" +html_show_sourcelink = ( + False # Remove 'view source code' from top of page (for html, not python) +) +numfig = True +bibtex_bibfiles = ['references.bib'] +bibtex_style = 'plain' diff --git a/doc/source/developer_guide/code_quality.rst b/doc/source/developer_guide/code_quality.rst new file mode 100644 index 000000000..1c02addbf --- /dev/null +++ b/doc/source/developer_guide/code_quality.rst @@ -0,0 +1,181 @@ +.. _code_quality: + +================= + Coding practice +================= + +Code quality assurance +====================== + +We use `Ruff `_, `mypy `_ and `Codespell `_ to maintain a high level of quality of the pelicun code. +Our objective is to always use the latest mature coding practice recommendations emerging from the Python community to ease adaptation to changes in its dependencies or even Python itself. + +We use the `numpy docstring style `_, and include comments in the code explaining the ideas behind various operations. +We are making an effort to use unambiguous variable and class/method/function names. +Especially for newer code, we are mindful of the complexity of methods/functions and break them down when they start to become too large, by extracting appropriate groups of lines and turning them into appropriately named hidden methods/functions. + +All code checking tools should be available when :ref:`installing pelicun under a developer setup `. +The most straight-forward way to run those tools is via the command-line. +All of the following commands are assumed to be executed from the package root directory (the one containing ``pyproject.toml``). + +Linting and formatting with Ruff +-------------------------------- + +.. code:: bash + + ruff check # Lint all files in the current directory. + ruff format # Format all files in the current directory. + +Ruff can automatically fix certain warnings when it is safe to do so. See also `Fixes `_. + +.. code:: + + ruff check --fix + +Warnings can also be automatically suppressed by adding #noqa directives. See `here `_. + +.. code:: bash + + ruff check --add-noqa + +Editor integration +.................. + +Like most code checkers, Ruff can be integrated with several editors to enable on-the-fly checks and auto-formatting. +Under `Editor Integration `_, their documentation describes the steps to enable this for VS Code, Neovim, Vim, Helix, Kate, Sublime Text, PyCharm, Emacs, TextMate, and Zed. + +Type checking with mypy +----------------------- + +Pelicun code is type hinted. +We use ``mypy`` for type checking. +Use the following command to type-check the code: + +.. code:: bash + + mypy pelicun --no-namespace-packages + +Type checking warnings can be silenced by adding ``#type: ignore`` at the lines that trigger them. +Please avoid silencing warnings in newly added code. + +Spell checking with Codespell +----------------------------- + +Codespell is a Python package used to check for common spelling mistakes in text files, particularly source code. It is available on PyPI, configured via pyproject.toml, and is executed as follows: + +.. code:: bash + + codespell . + +False positives can be placed in a dedicated file (we currently call it ``ignore_words.txt``) to be ignored. +Please avoid using variable names that trigger codespell. +This is easy when variable names are long and explicit. +E.g., ``response_spectrum_target`` instead of ``resptr``. + +Unit tests +========== + +We use `pytest `_ to write unit tests for pelicun. +The tests can be executed with the following command. + +.. code:: bash + + python -m pytest pelicun/tests --cov=pelicun --cov-report html + +When the test runs finish, visit ``htmlcov/index.html`` for a comprehensive view of code coverage. + +The tests can be debugged like any other Python code, by inserting ``breakpoint()`` at any line and executing the line above. +When the breakpoint is reached you will gain access to ``PDB``, the Python Debugger. + +Please extend the test suite whenever you introduce new pelicun code, and update it if needed when making changes to the existing code. +Avoid opening pull requests with changes that reduce code coverage by not writing tests for your code. + +Documentation +============= + +We use `sphinx `_ with the `Read the Docs theme `_ to generate our documentation pages. + +We use the following extensions: + +- `nbsphinx `_ to integrate jupyter notebooks into the documentation, particularly for the pelicun examples. + In the source code they are stored as python files with special syntax defining individual cells, and we use `jupytext `_ to automatically turn them into notebooks when the documentation is compiled (see ``nbsphinx_custom_formats`` in ``conf.py``). + +- `Sphinx design `_ for cards and drop-downs. + +- `sphinx.ext.mathjax `_ for math support. + +- `sphinx.ext.doctest `_ to actively test examples included in docstrings. + +- `numpydoc `_ and `autodoc `_ to generate the API documentation from the docstrings in the source code. + +- `sphinx.ext.autosummary `_ for the API docs. + +- `sphinx.ext.viewcode `_ to add links that point to the source code in the API docs. + +- `sphinx.ext.intersphinx `_ to link to other projects' documentation. + +- `sphinx.ext.githubpages `_ for publishing in GitHub pages. + + +Building the documentation +-------------------------- + +To build the documentation, navigate to `doc` and run the following command: + +.. tab-set:: + + .. tab-item:: Linux & Mac + + .. code:: bash + + make html + + .. tab-item:: Windows + + .. code:: bash + + .\make.bat html + +To see more options: + +.. tab-set:: + + .. tab-item:: Linux & Mac + + .. code:: bash + + make + + .. tab-item:: Windows + + .. code:: bash + + .\make.bat + + +Extending the documentation +--------------------------- + +Extending the documentation can be done in several ways: + +- By adding content to ``.rst`` files or adding more such files. + See the structure of the ``doc/source`` directory and look at the ``index.rst`` files to gain familiarity with the structure of the documentation. + When a page is added it will need to be included in a ``toctree`` directive in order for it to be registered and have a way of being accessed. +- By adding or modifying example notebooks under ``doc/examples/notebooks``. + When a new notebook is added, it needs to be included in ``doc/examples/index.rst``. + Please review that index file and how other notebooks are listed to become familiar with our approach. + +After making a change you can simply rebuild the documentation with the command above. +Once the documentation pages are built, please verify no Sphinx warnings are reported. +A warning count is shown after "build succeeded", close to the last output line: + +.. code-block:: none + :emphasize-lines: 3 + + [...] + dumping object inventory... done + build succeeded, 1 warning. + + The HTML pages are in build/html. + +If there are warnings, please address them before contributing your changes. diff --git a/doc/source/developer_guide/development_environment.rst b/doc/source/developer_guide/development_environment.rst new file mode 100644 index 000000000..409e939c5 --- /dev/null +++ b/doc/source/developer_guide/development_environment.rst @@ -0,0 +1,27 @@ +.. _development_environment: + +Setting up a development environment +------------------------------------ + +.. tip:: + + We recommend creating a dedicated `virtual environment `_ for your pelicun development environment. + See also `conda `_ and `mamba `_, two widely used programs featuring environment management. + +Clone the repository:: + + git clone --recurse-submodules https://github.com/NHERI-SimCenter/pelicun + +Pelicun uses the SimCenter `DamageAndLossModelLibrary `_ as a submodule. +In the above, ``recurse-submodules`` ensures that all files of that repository are also obtained. + +.. tip:: + + If you are planning to contribute code, please `fork the repository `_ and clone your own fork. + + +Install pelicun in editable mode with the following command issued from the package's root directory:: + + python -m pip install -e .[development] + +This will install pelicun in editable mode as well as all dependencies needed for development. diff --git a/doc/source/developer_guide/getting_started.rst b/doc/source/developer_guide/getting_started.rst new file mode 100644 index 000000000..20e38a087 --- /dev/null +++ b/doc/source/developer_guide/getting_started.rst @@ -0,0 +1,224 @@ +Welcome to the pelicun developer guide. +The following pages contain information on setting up a development environment, our code quality requirements, our code of conduct, and information on the submission process. +Thank you for your interest in extending pelicun. +We are looking forward to your contributions! + +=============== +Getting started +=============== + +Code of conduct +=============== + +Our pledge +---------- + +We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socioeconomic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. + +Our standards +------------- + +Examples of behavior that contributes to a positive environment for our community include: + +- Demonstrating empathy and kindness toward other people. +- Being respectful of differing opinions, viewpoints, and experiences. +- Giving and gracefully accepting constructive feedback. +- Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience. +- Focusing on what is best not just for us as individuals, but for the overall community. + +Examples of unacceptable behavior include: + +- The use of sexualized language or imagery, and sexual attention or advances of any kind. +- Trolling, insulting or derogatory comments, and personal or political attacks. +- Public or private harassment. +- Publishing others’ private information, such as a physical or email address, without their explicit permission. +- Other conduct which could reasonably be considered inappropriate in a professional setting. + +Enforcement responsibilities +---------------------------- + +Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. + +Scope +----- + +This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. +Examples of representing our community include using an official email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. + +Enforcement +----------- + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leader responsible for enforcement at ``adamzs@stanford.edu``. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the reporter of any incident. + +Enforcement guidelines +---------------------- + +Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: + +1. Correction +~~~~~~~~~~~~~ + +**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. +A public apology may be requested. + +2. Warning +~~~~~~~~~~ + +**Community Impact**: A violation through a single incident or series of actions. + +**Consequence**: A warning with consequences for continued behavior. +No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. +This includes avoiding interactions in community spaces as well as external channels like social media. +Violating these terms may lead to a temporary or permanent ban. + +3. Temporary ban +~~~~~~~~~~~~~~~~ + +**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. +No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +4. Permanent Ban +~~~~~~~~~~~~~~~~ + +**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the community. + +Attribution +----------- + +This Code of Conduct is adapted from the `Contributor Covenant `__, version 2.1, available at https://www.contributor-covenant.org/version/2/1/code_of_conduct.html. + +Community Impact Guidelines were inspired by `Mozilla’s code of conduct enforcement ladder `__. + +For answers to common questions about this code of conduct, see the FAQ at https://www.contributor-covenant.org/faq. +Translations are available at https://www.contributor-covenant.org/translations. + +.. _contributing: + +How to contribute +================= + +Prerequisites +------------- + +Contributing to pelicun requires being familiar with the following: + +.. dropdown:: Python Programming + + Being familiar with object-oriented programming in Python, the PDB debugger, and having familiarity with Numpy and Pandas to handle arrays and DataFrames. + + The following resources may be helpful: + + - `python.org tutorial `_ + - `numpy beginner's guide `_ + - `numpy user guide `_ + - `pandas beginner's guide `_ + - `pandas user guide `_ + +.. dropdown:: Virtual Environments + + Managing a development environment, installing and removing packages. + + The following resources may be helpful: + + - `Python: Virtual Environments and Packages `_ + + - `Conda: Managing environments `_ + - `Micromamba User Guide `_ + +.. dropdown:: reStructured Text Markup + + Being able to extend the documentation by reviewing the existing files and following the same pattern, without introducing compilation warnings. + + The following resources may be helpful: + + - `reStructuredText documentation `_ + - `Sphinx User Guide: Using Sphinx `_ + +.. dropdown:: Git for Version Control + + Knowing how to clone a repository, create and checkout branches, review commit logs, commit well-documented changes, or stashing them for later use. + The following may be useful: + + `git reference manual `_ + +.. dropdown:: Command Line + + Being able to set up and call command-line programs beyond Git, including the linting and formatting tools we use. + + The following resources may be useful: + + .. tab-set:: + + .. tab-item:: Linux & Mac + + `Bash Reference Manual `_ + + .. tab-item:: Windows + + `PowerShell Documentation `_ + +.. dropdown:: Pattern-matching + + Ultimately, we learn by example. + The files already present in the pelicun source code offer an existing template that can help you understand what any potential additions should look like. + Actively exploring the existing files, tinkering them and breaking things is a great way to gain a deeper understanding of the package. + +Contributing workflow +--------------------- + +The development of pelicun is done via Pull Requests (PR) on GitHub. +Contributors need to carry out the following steps to submit a successful PR: + +- `Create a GitHub account `_, if you don't already have one. +- `Fork the primary pelicun repository `_. +- On the fork, create a feature branch with an appropriate starting point. +- Make and commit changes to the branch. +- Push to your remote repository. +- Open a well-documented PR on the GitHub website. + +If you are working on multiple features, please use multiple dedicated feature branches with the same starting point instead of lumping them into a single branch. +This approach substantially simplifies the review process, and changes on multiple fronts can be easily merged after being reviewed. +On each feature branch, please commit changes often and include meaningful commit titles and messages. + +.. tip:: + + Consider taking advantage of advanced Git clients, which enable selective, partial staging of hunks, helping organize commits. + + .. dropdown:: Potential options + + - `Emacs Magit `_, for Emacs users. Tried and true, used by our development team. + - `Sublime Merge `_, also used by our development team. + - `GitHub Desktop `_, convenient and user-friendly. + +Code review process +------------------- + +After you submit your PR, we are going to promptly review your commits, offer feedback and request changes. +All contributions code need to be comprehensive. +That is, inclusion of new objects, methods, or functions should be accompanied by unit tests having reasonable coverage, and extension of the documentation pages as appropriate. +We will direct you to extend your changes to cover those areas if you haven't done so. +After the review process, the PR will either be merged to the main repository or rejected with sufficient justification. + +We will accept any contribution that we believe ultimately improves pelicun, no matter how big or small. +You are welcome to open a PR even for a single typo. + +Identifying contribution opportunities +-------------------------------------- + +The `Issues `_ page on GitHub documents areas needing improvement. +If you are interested in becoming a contributor but don't have a specific change in mind, feel free to work on addressing any of the issues listed. +If you would like to offer a contribution that extends the fundamental framework, please begin by `initiating a discussion `_ before you work on changes to avoid making unnecessary effort. diff --git a/doc/source/developer_guide/internals.rst b/doc/source/developer_guide/internals.rst new file mode 100644 index 000000000..d1c9cb048 --- /dev/null +++ b/doc/source/developer_guide/internals.rst @@ -0,0 +1,49 @@ +.. _internals: + +Package architecture +-------------------- + +Overview of files +................. + ++-------------------+---------------------------------------+ +|Path |Description | ++===================+=======================================+ +|``pelicun/`` |Main package source code. | ++-------------------+---------------------------------------+ +|``doc/`` |Documentation source code. | ++-------------------+---------------------------------------+ +|``.github/`` |GitHub-related workflow files. | ++-------------------+---------------------------------------+ +|``pyproject.toml`` |Main configuration file. | ++-------------------+---------------------------------------+ +|``setup.py`` |Package setup file. | ++-------------------+---------------------------------------+ +|``MANIFEST.in`` |Defines files to include when building | +| |the package. | ++-------------------+---------------------------------------+ + +.. note:: + + We are currently in the process of migrating most configuration files to ``pyproject.toml``. + +We use `setuptools `_ to build the package, using ``setup.py`` for configuration. +In ``setup.py`` we define an entry point called ``pelicun``, directing to the ``main`` method of ``DL_calculation.py``, used to run pelicun from the command line. + +The python source code and unit tests are located under ``pelicun/``. +``assessment.py`` is the primary file defining assessment classes and methods. +Modules under ``model/`` contain various models used by ``Assessment`` objects. +Such models handle the representation of asset inventories, as well as demand, damage, and loss samples. +``base.py`` defines commonly used objects. +``file_io.py`` defines methods related to reading and writing to files. +``uq.py`` defines classes and methods used for uncertainty quantification, including random variable objects and registries, and parameter recovery methods used to fit distributions to raw data samples. +``warnings.py`` defines custom errors and warnings used in pelicun. +``tools/DL_calculation.py`` enables the invocation of analyses using the command line. +``settings/`` contains several ``JSON`` files used to define default units, configuration options and input validation. +``resources/`` contains default damage and loss model parameters. + +.. tip:: + + For a detailed overview of these files, please see the `API documentation <../api_reference/index.rst>`_ or directly review the source code. + + A direct way to become familiar with an area of the source code you are interested in working with is to debug an applicable example or test and follow through the calculation steps involved, taking notes in the process. diff --git a/doc/source/examples/index.rst b/doc/source/examples/index.rst new file mode 100644 index 000000000..5c7577d2d --- /dev/null +++ b/doc/source/examples/index.rst @@ -0,0 +1,42 @@ +:notoc: + +.. _examples: + +******** +Examples +******** + +Pelicun examples are listed in the following index. + +.. attention:: + + These documentation pages are brand new and in active development. + Increasing the set of examples is a very high priority. + Please come back soon! + ++-----------+---------------------------------------------------------+ +|Example |Description | ++===========+=========================================================+ +|`E1`_ |Example of a simple assessment involving a single loss | +| |function. The example and associated input files can be | +| |easily extended to include more loss functions and input | +| |demands. | ++-----------+---------------------------------------------------------+ +|`E2`_ |Example validating the estimated damage state | +| |probabilities of a single component. | ++-----------+---------------------------------------------------------+ +|`E3`_ |A loss assessment combining fragility-based damage | +| |consequences and loss functions. | ++-----------+---------------------------------------------------------+ + +.. toctree:: + :maxdepth: 1 + :hidden: + + notebooks/example_1.pct.py + notebooks/example_2.pct.py + notebooks/example_3.pct.py + +.. _E1: notebooks/example_1.pct.py +.. _E2: notebooks/example_2.pct.py +.. _E3: notebooks/example_3.pct.py diff --git a/doc/source/examples/notebooks/example_1.pct.py b/doc/source/examples/notebooks/example_1.pct.py new file mode 100644 index 000000000..a67224c47 --- /dev/null +++ b/doc/source/examples/notebooks/example_1.pct.py @@ -0,0 +1,99 @@ +# %% [markdown] +"""# Example 1: Simple loss function.""" + +# %% [markdown] +""" +In this example, a single loss function is defined as a 1:1 mapping of the input EDP. +This means that the resulting loss distribution will be the same as the EDP distribution, allowing us to test and confirm that this is what happens. +""" + +# %% +from __future__ import annotations + +import numpy as np +import pandas as pd + +from pelicun import assessment, file_io + +# %% +sample_size = 100000 + +# %% +# initialize a pelicun assessment +asmnt = assessment.Assessment({'PrintLog': False, 'Seed': 42}) + +# %% + +# +# Demands +# + +demands = pd.DataFrame( + { + 'Theta_0': [0.50], + 'Theta_1': [0.90], + 'Family': ['lognormal'], + 'Units': ['mps2'], + }, + index=pd.MultiIndex.from_tuples( + [ + ('PFA', '0', '1'), + ], + ), +) + +asmnt.demand.load_model({'marginals': demands}) + +asmnt.demand.generate_sample({'SampleSize': sample_size}) + +# +# Asset +# + +asmnt.stories = 1 + +cmp_marginals = pd.read_csv('example_1/CMP_marginals.csv', index_col=0) +cmp_marginals['Blocks'] = cmp_marginals['Blocks'] +asmnt.asset.load_cmp_model({'marginals': cmp_marginals}) + +asmnt.asset.generate_cmp_sample(sample_size) + +# %% +# +# Damage +# + +# nothing to do here. + +# %% +# +# Losses +# + +asmnt.loss.decision_variables = ('Cost',) + +loss_map = pd.DataFrame(['cmp.A'], columns=['Repair'], index=['cmp.A']) +asmnt.loss.add_loss_map(loss_map) + +loss_functions = file_io.load_data( + 'example_1/loss_functions.csv', + reindex=False, + unit_conversion_factors=asmnt.unit_conversion_factors, +) +# %% nbsphinx="hidden" +assert isinstance(loss_functions, pd.DataFrame) +# %% +asmnt.loss.load_model_parameters([loss_functions]) +asmnt.loss.calculate() + +loss, _ = asmnt.loss.aggregate_losses(future=True) +# %% nbsphinx="hidden" +assert isinstance(loss, pd.DataFrame) + +loss_vals = loss['repair_cost'].to_numpy() + +# %% nbsphinx="hidden" +# sample median should be close to 0.05 +assert np.allclose(np.median(loss_vals), 0.05, atol=1e-2) +# dispersion should be close to 0.9 +assert np.allclose(np.log(loss_vals).std(), 0.90, atol=1e-2) diff --git a/pelicun/tests/validation/0/data/CMP_marginals.csv b/doc/source/examples/notebooks/example_1/CMP_marginals.csv similarity index 100% rename from pelicun/tests/validation/0/data/CMP_marginals.csv rename to doc/source/examples/notebooks/example_1/CMP_marginals.csv diff --git a/pelicun/tests/validation/0/data/loss_functions.csv b/doc/source/examples/notebooks/example_1/loss_functions.csv similarity index 100% rename from pelicun/tests/validation/0/data/loss_functions.csv rename to doc/source/examples/notebooks/example_1/loss_functions.csv diff --git a/doc/source/examples/notebooks/example_2.pct.py b/doc/source/examples/notebooks/example_2.pct.py new file mode 100644 index 000000000..4165e7ce3 --- /dev/null +++ b/doc/source/examples/notebooks/example_2.pct.py @@ -0,0 +1,164 @@ +# %% [markdown] +r""" +# Example 2: Damage state validation. + +Validation test for the probability of each damage state of a +component. + +Here we test whether we get the correct damage state probabilities for +a single component with two damage states. +For such a component, assuming the EDP demand and the fragility curve +capacities are all lognormal, there is a closed-form solution for the +probability of each damage state. +We utilize those equations to ensure that the probabilities obtained +from our Monte-Carlo sample are in line with our expectations. + +If $\mathrm{Y} \sim \textrm{LogNormal}(\delta, \beta)$, +then $\mathrm{X} = \log(\mathrm{Y}) \sim \textrm{Normal}(\mu, \sigma)$ with +$\mu = \log(\delta)$ and $\sigma = \beta$. + +$$ +\begin{align*} +\mathrm{P}(\mathrm{DS}=0) &= 1 - \Phi\left(\frac{\log(\delta_D) - \log(\delta_{C1})}{\sqrt{\beta_{D}^2 + \beta_{C1}^2}}\right), \\ +\mathrm{P}(\mathrm{DS}=1) &= \Phi\left(\frac{\log(\delta_D) - \log(\delta_{C1})}{\sqrt{\beta_D^2 + \beta_{C1}^2}}\right) - \Phi\left(\frac{\log(\delta_{D}) - \log(\delta_{C2})}{\sqrt{\beta_D^2 + \beta_{C2}^2}}\right), \\ +\mathrm{P}(\mathrm{DS}=2) &= \Phi\left(\frac{\log(\delta_D) - \log(\delta_{C2})}{\sqrt{\beta_D^2 + \beta_{C2}^2}}\right), \\ +\end{align*} +$$ +where $\Phi$ is the cumulative distribution function of the standard normal distribution, +$\delta_{C1}$, $\delta_{C2}$, $\beta_{C1}$, $\beta_{C2}$ are the medians and dispersions of the +fragility curve capacities, and $\delta_{D}$, $\beta_{D}$ is +the median and dispersion of the EDP demand. + +The equations inherently assume that the capacity RVs for the damage +states are perfectly correlated, which is the case for sequential +damage states. + +""" + +# %% +from __future__ import annotations + +import tempfile + +import numpy as np +import pandas as pd +from scipy.stats import norm # type: ignore + +from pelicun import assessment, file_io + +# %% +sample_size = 1000000 + +asmnt = assessment.Assessment({'PrintLog': False, 'Seed': 42}) + +# %% +# +# Demands +# + +demands = pd.DataFrame( + { + 'Theta_0': [0.015], + 'Theta_1': [0.60], + 'Family': ['lognormal'], + 'Units': ['rad'], + }, + index=pd.MultiIndex.from_tuples( + [ + ('PID', '1', '1'), + ], + ), +) + +# load the demand model +asmnt.demand.load_model({'marginals': demands}) + +# generate samples +asmnt.demand.generate_sample({'SampleSize': sample_size}) + +# %% +# +# Asset +# + +# specify number of stories +asmnt.stories = 1 + +# load component definitions +cmp_marginals = pd.read_csv('example_2/CMP_marginals.csv', index_col=0) +cmp_marginals['Blocks'] = cmp_marginals['Blocks'] +asmnt.asset.load_cmp_model({'marginals': cmp_marginals}) + +# generate sample +asmnt.asset.generate_cmp_sample(sample_size) + +# %% +# +# Damage +# + +damage_db = file_io.load_data( + 'example_2/damage_db.csv', + reindex=False, + unit_conversion_factors=asmnt.unit_conversion_factors, +) +assert isinstance(damage_db, pd.DataFrame) + +cmp_set = set(asmnt.asset.list_unique_component_ids()) + +# load the models into pelicun +asmnt.damage.load_model_parameters([damage_db], cmp_set) + +# calculate damages +asmnt.damage.calculate() + +probs = asmnt.damage.ds_model.probabilities() + +# %% +# +# Analytical calculation of the probability of each damage state +# + +demand_median = 0.015 +demand_beta = 0.60 +capacity_1_median = 0.015 +capacity_2_median = 0.02 +capacity_beta = 0.50 + +# If Y is LogNormal(delta, beta), then X = Log(Y) is Normal(mu, sigma) +# with mu = log(delta) and sigma = beta +demand_mean = np.log(demand_median) +capacity_1_mean = np.log(capacity_1_median) +capacity_2_mean = np.log(capacity_2_median) +demand_std = demand_beta +capacity_std = capacity_beta + +p0 = 1.00 - norm.cdf( + (demand_mean - capacity_1_mean) / np.sqrt(demand_std**2 + capacity_std**2) +) +p1 = norm.cdf( + (demand_mean - capacity_1_mean) / np.sqrt(demand_std**2 + capacity_std**2) +) - norm.cdf( + (demand_mean - capacity_2_mean) / np.sqrt(demand_std**2 + capacity_std**2) +) +p2 = norm.cdf( + (demand_mean - capacity_2_mean) / np.sqrt(demand_std**2 + capacity_std**2) +) + +assert np.allclose(probs.iloc[0, 0], p0, atol=1e-2) # type: ignore +assert np.allclose(probs.iloc[0, 1], p1, atol=1e-2) # type: ignore +assert np.allclose(probs.iloc[0, 2], p2, atol=1e-2) # type: ignore + +# %% nbsphinx="hidden" +# +# Also test load/save sample +# + +assert asmnt.damage.ds_model.sample is not None +asmnt.damage.ds_model.sample = asmnt.damage.ds_model.sample.iloc[0:100, :] +# (we reduce the number of realizations to conserve resources) +before = asmnt.damage.ds_model.sample.copy() +temp_dir = tempfile.mkdtemp() +asmnt.damage.save_sample(f'{temp_dir}/mdl.csv') +asmnt.damage.load_sample(f'{temp_dir}/mdl.csv') +pd.testing.assert_frame_equal(before, asmnt.damage.ds_model.sample) diff --git a/pelicun/tests/validation/1/data/CMP_marginals.csv b/doc/source/examples/notebooks/example_2/CMP_marginals.csv similarity index 100% rename from pelicun/tests/validation/1/data/CMP_marginals.csv rename to doc/source/examples/notebooks/example_2/CMP_marginals.csv diff --git a/pelicun/tests/validation/1/data/damage_db.csv b/doc/source/examples/notebooks/example_2/damage_db.csv similarity index 100% rename from pelicun/tests/validation/1/data/damage_db.csv rename to doc/source/examples/notebooks/example_2/damage_db.csv diff --git a/doc/source/examples/notebooks/example_3.pct.py b/doc/source/examples/notebooks/example_3.pct.py new file mode 100644 index 000000000..65c73ccc1 --- /dev/null +++ b/doc/source/examples/notebooks/example_3.pct.py @@ -0,0 +1,216 @@ +# %% [markdown] +""" +# Example 3: Combining fragility-based damage consequences and loss functions. + +Tests a complete loss estimation workflow combining damage state and +loss function driven components. The code is based on PRJ-3411v5 +hosted on DesignSafe. + +""" + +# %% +import tempfile + +import numpy as np +import pandas as pd +import pytest + +import pelicun +from pelicun import assessment, file_io +from pelicun.pelicun_warnings import PelicunWarning + +# %% +temp_dir = tempfile.mkdtemp() + +sample_size = 10000 + +# %% +# Initialize a pelicun assessment +asmnt = assessment.Assessment( + {'PrintLog': True, 'Seed': 415, 'LogFile': f'{temp_dir}/log_file.txt'} +) + +asmnt.options.list_all_ds = True +asmnt.options.eco_scale['AcrossFloors'] = True +asmnt.options.eco_scale['AcrossDamageStates'] = True + +# %% +demand_data = file_io.load_data( + 'example_3/demand_data.csv', + unit_conversion_factors=None, + reindex=False, +) +ndims = len(demand_data) +perfect_correlation = pd.DataFrame( + np.ones((ndims, ndims)), + columns=demand_data.index, # type: ignore + index=demand_data.index, # type: ignore +) + +# %% +# +# Additional damage state-driven components +# + +damage_db = pelicun.file_io.load_data( + 'example_3/additional_damage_db.csv', + reindex=False, + unit_conversion_factors=asmnt.unit_conversion_factors, +) +consequences = pelicun.file_io.load_data( + 'example_3/additional_consequences.csv', + reindex=False, + unit_conversion_factors=asmnt.unit_conversion_factors, +) + +# %% +# +# Additional loss function-driven components +# + +loss_functions = pelicun.file_io.load_data( + 'example_3/additional_loss_functions.csv', + reindex=False, + unit_conversion_factors=asmnt.unit_conversion_factors, +) + +# %% +# +# Demands +# + +# Load the demand model +asmnt.demand.load_model( + {'marginals': demand_data, 'correlation': perfect_correlation} +) + +# Generate samples +asmnt.demand.generate_sample({'SampleSize': sample_size}) + + +def add_more_edps() -> None: + """Add SA_1.13 and residual drift to the demand sample.""" + # Add residual drift and Sa + demand_sample = asmnt.demand.save_sample() + + # RIDs are all fixed for testing. + rid = pd.concat( + [ + pd.DataFrame( + np.full(demand_sample['PID'].shape, 0.0050), # type: ignore + index=demand_sample['PID'].index, # type: ignore + columns=demand_sample['PID'].columns, # type: ignore + ) + ], + axis=1, + keys=['RID'], + ) + demand_sample_ext = pd.concat([demand_sample, rid], axis=1) # type: ignore + + demand_sample_ext['SA_1.13', 0, 1] = 1.50 + + # Add units to the data + demand_sample_ext.T.insert(0, 'Units', '') + + # PFA and SA are in "g" in this example, while PID and RID are "rad" + demand_sample_ext.loc['Units', ['PFA', 'SA_1.13']] = 'g' + demand_sample_ext.loc['Units', ['PID', 'RID']] = 'rad' + + asmnt.demand.load_sample(demand_sample_ext) + + +add_more_edps() + +# %% +# +# Asset +# + +# Specify number of stories +asmnt.stories = 1 + +# Load component definitions +cmp_marginals = pd.read_csv('example_3/CMP_marginals.csv', index_col=0) +cmp_marginals['Blocks'] = cmp_marginals['Blocks'] +asmnt.asset.load_cmp_model({'marginals': cmp_marginals}) + +# Generate sample +asmnt.asset.generate_cmp_sample(sample_size) + +# %% +# +# Damage +# + +cmp_set = set(asmnt.asset.list_unique_component_ids()) + +# Load the models into pelicun +asmnt.damage.load_model_parameters( + [ + damage_db, # type: ignore + 'PelicunDefault/damage_DB_FEMA_P58_2nd.csv', + ], + cmp_set, +) + +# Prescribe the damage process +dmg_process = { + '1_collapse': {'DS1': 'ALL_NA'}, + '2_excessiveRID': {'DS1': 'irreparable_DS1'}, +} + +# Calculate damages + +asmnt.damage.calculate(dmg_process=dmg_process) + +# Test load sample, save sample +asmnt.damage.save_sample(f'{temp_dir}/out.csv') +asmnt.damage.load_sample(f'{temp_dir}/out.csv') + +# %% nbsphinx="hidden" +assert asmnt.damage.ds_model.sample is not None +# %% +asmnt.damage.ds_model.sample.mean() + +# %% +# +# Losses +# + +# Create the loss map +loss_map = pd.DataFrame( + ['replacement', 'replacement'], + columns=['Repair'], + index=['collapse', 'irreparable'], +) + +# Load the loss model +asmnt.loss.decision_variables = ('Cost', 'Time') +asmnt.loss.add_loss_map(loss_map, loss_map_policy='fill') +with pytest.warns(PelicunWarning): + asmnt.loss.load_model_parameters( + [ + consequences, # type: ignore + loss_functions, # type: ignore + 'PelicunDefault/loss_repair_DB_FEMA_P58_2nd.csv', + ] + ) + +# Perform the calculation +asmnt.loss.calculate() + +# Test load sample, save sample +with pytest.warns(PelicunWarning): + asmnt.loss.save_sample(f'{temp_dir}/sample.csv') + asmnt.loss.load_sample(f'{temp_dir}/sample.csv') + +# +# Loss sample aggregation +# + +# Get the aggregated losses +with pytest.warns(PelicunWarning): + agg_df = asmnt.loss.aggregate_losses() + +# %% nbsphinx="hidden" +assert agg_df is not None diff --git a/pelicun/tests/validation/2/data/CMP_marginals.csv b/doc/source/examples/notebooks/example_3/CMP_marginals.csv similarity index 100% rename from pelicun/tests/validation/2/data/CMP_marginals.csv rename to doc/source/examples/notebooks/example_3/CMP_marginals.csv diff --git a/pelicun/tests/validation/2/data/additional_consequences.csv b/doc/source/examples/notebooks/example_3/additional_consequences.csv similarity index 100% rename from pelicun/tests/validation/2/data/additional_consequences.csv rename to doc/source/examples/notebooks/example_3/additional_consequences.csv diff --git a/pelicun/tests/validation/2/data/additional_damage_db.csv b/doc/source/examples/notebooks/example_3/additional_damage_db.csv similarity index 100% rename from pelicun/tests/validation/2/data/additional_damage_db.csv rename to doc/source/examples/notebooks/example_3/additional_damage_db.csv diff --git a/pelicun/tests/validation/2/data/additional_loss_functions.csv b/doc/source/examples/notebooks/example_3/additional_loss_functions.csv similarity index 100% rename from pelicun/tests/validation/2/data/additional_loss_functions.csv rename to doc/source/examples/notebooks/example_3/additional_loss_functions.csv diff --git a/pelicun/tests/validation/2/data/demand_data.csv b/doc/source/examples/notebooks/example_3/demand_data.csv similarity index 100% rename from pelicun/tests/validation/2/data/demand_data.csv rename to doc/source/examples/notebooks/example_3/demand_data.csv diff --git a/pelicun/tests/validation/2/data/loss_functions.csv b/doc/source/examples/notebooks/example_3/loss_functions.csv similarity index 100% rename from pelicun/tests/validation/2/data/loss_functions.csv rename to doc/source/examples/notebooks/example_3/loss_functions.csv diff --git a/doc/source/examples/notebooks/example_4/CMP_marginals.csv b/doc/source/examples/notebooks/example_4/CMP_marginals.csv new file mode 100755 index 000000000..e085a1f00 --- /dev/null +++ b/doc/source/examples/notebooks/example_4/CMP_marginals.csv @@ -0,0 +1,8 @@ +,Units,Location,Direction,Theta_0,Blocks,Comment +D.30.31.013i,ea,roof,0,1,,Chiller (parameters have been modified for testing) +missing.component,ea,0,1,1,,Testing +testing.component,ea,1,1,1,,Testing +testing.component.2,ea,1,1,2,4,Testing +collapse,ea,0,1,1,,Collapsed building +excessiveRID,ea,all,"1,2",1,,Excessive residual drift +irreparable,ea,0,1,1,,Irreparable building diff --git a/doc/source/examples/notebooks/example_4/additional_consequences.csv b/doc/source/examples/notebooks/example_4/additional_consequences.csv new file mode 100644 index 000000000..889ca6ae9 --- /dev/null +++ b/doc/source/examples/notebooks/example_4/additional_consequences.csv @@ -0,0 +1,3 @@ +-,Incomplete-,Quantity-Unit,DV-Unit,DS1-Theta_0 +replacement-Cost,0,1 EA,USD_2011,21600000 +replacement-Time,0,1 EA,worker_day,12500 diff --git a/doc/source/examples/notebooks/example_4/additional_damage_db.csv b/doc/source/examples/notebooks/example_4/additional_damage_db.csv new file mode 100644 index 000000000..5d7c885ac --- /dev/null +++ b/doc/source/examples/notebooks/example_4/additional_damage_db.csv @@ -0,0 +1,5 @@ +ID,Demand-Directional,Demand-Offset,Demand-Type,Demand-Unit,Incomplete-,LS1-DamageStateWeights,LS1-Family,LS1-Theta_0,LS1-Theta_1,LS2-DamageStateWeights,LS2-Family,LS2-Theta_0,LS2-Theta_1,LS3-DamageStateWeights,LS3-Family,LS3-Theta_0,LS3-Theta_1,LS4-DamageStateWeights,LS4-Family,LS4-Theta_0,LS4-Theta_1 +D.30.31.013i,0.0,0.0,Peak Floor Acceleration,g,0,0.340000 | 0.330000 | 0.330000,lognormal,0.40,0.5,,,,,,,,,,,, +excessiveRID,1.0,0.0,Residual Interstory Drift Ratio,rad,0,,lognormal,0.01,0.3,,,,,,,,,,,, +irreparable,1.0,0.0,Peak Spectral Acceleration|1.13,g,0,,,10000000000.0,,,,,,,,,,,,, +collapse,1.0,0.0,Peak Spectral Acceleration|1.13,g,0,,lognormal,1.50,0.5,,,,,,,,,,,, diff --git a/doc/source/examples/notebooks/example_4/additional_loss_functions.csv b/doc/source/examples/notebooks/example_4/additional_loss_functions.csv new file mode 100644 index 000000000..7d9bf4b93 --- /dev/null +++ b/doc/source/examples/notebooks/example_4/additional_loss_functions.csv @@ -0,0 +1,5 @@ +-,DV-Unit,Demand-Directional,Demand-Offset,Demand-Type,Demand-Unit,LossFunction-Theta_0,LossFunction-Theta_1,LossFunction-Family +testing.component-Cost,USD_2011,0,0,Peak Floor Acceleration,g,"0.00,100000.00|0.00,5.00",0.3,lognormal +testing.component-Time,USD_2011,0,0,Peak Floor Acceleration,g,"0.00,50.00|0.00,5.00",0.3,lognormal +testing.component.2-Cost,USD_2011,0,0,Peak Floor Acceleration,g,"0.00,10.00|0.00,5.00",, +testing.component.2-Time,USD_2011,0,0,Peak Floor Acceleration,g,"0.00,5.00|0.00,5.00",, diff --git a/doc/source/examples/notebooks/example_4/demand_data.csv b/doc/source/examples/notebooks/example_4/demand_data.csv new file mode 100644 index 000000000..7f2efa3f0 --- /dev/null +++ b/doc/source/examples/notebooks/example_4/demand_data.csv @@ -0,0 +1,19 @@ +--,Units,Family,Theta_0,Theta_1 +PFA-0-1,g,lognormal,0.45,0.40 +PFA-0-2,g,lognormal,0.45,0.40 +PFA-1-1,g,lognormal,0.45,0.40 +PFA-1-2,g,lognormal,0.45,0.40 +PFA-2-1,g,lognormal,0.45,0.40 +PFA-2-2,g,lognormal,0.45,0.40 +PFA-3-1,g,lognormal,0.45,0.40 +PFA-3-2,g,lognormal,0.45,0.40 +PFA-4-1,g,lognormal,0.45,0.40 +PFA-4-2,g,lognormal,0.45,0.40 +PID-1-1,rad,lognormal,0.03,0.35 +PID-1-2,rad,lognormal,0.03,0.35 +PID-2-1,rad,lognormal,0.03,0.35 +PID-2-2,rad,lognormal,0.03,0.35 +PID-3-1,rad,lognormal,0.03,0.35 +PID-3-2,rad,lognormal,0.03,0.35 +PID-4-1,rad,lognormal,0.03,0.35 +PID-4-2,rad,lognormal,0.03,0.35 diff --git a/doc/source/examples/notebooks/example_4/loss_functions.csv b/doc/source/examples/notebooks/example_4/loss_functions.csv new file mode 100644 index 000000000..23c7b1939 --- /dev/null +++ b/doc/source/examples/notebooks/example_4/loss_functions.csv @@ -0,0 +1,2 @@ +-,DV-Unit,Demand-Directional,Demand-Offset,Demand-Type,Demand-Unit,LossFunction-Theta_0,LossFunction-Theta_1,LossFunction-Family +cmp.A-Cost,loss_ratio,1,0,Peak Floor Acceleration,g,"0.00,1000.00|0.00,1000.00",, diff --git a/doc/source/examples/notebooks/template.pct.py.txt b/doc/source/examples/notebooks/template.pct.py.txt new file mode 100644 index 000000000..3ed62b177 --- /dev/null +++ b/doc/source/examples/notebooks/template.pct.py.txt @@ -0,0 +1,36 @@ +# %% [markdown] +""" +# First-level section title goes here. + +This is a Markdown cell that uses multiline comments. +Second line here. + +## Second level section + +Here is some math: +$$ + \int_0^\infty \frac{x^3}{e^x-1}\,dx = \frac{\pi^4}{15} +$$ + +And here is some inline math. Variable $x$ is equal to $y$. +""" + +# %% [markdown] +# Another Markdown cell with a single-line comment. + + +# %% nbsphinx="hidden" +# This is a hidden code cell +# We will use this to turn the examples into additional tests without +# polluting the documentation with imports and assertions. +class A: + def one(): + return 1 + + def two(): + return 2 + + +# %% +# This is a visible code cell +print("Hello, world!") diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 000000000..bc0c9547e --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,49 @@ +:notoc: + +======================= + Pelicun Documentation +======================= + +.. warning:: + + Development Preview Only. + This documentation is intended for internal review and demonstration purposes. + It is not finalized or ready for public use. + +Pelicun is an open-source Python package for the probabilistic estimation of losses, injuries, and community resilience under natural disasters. +Community-driven, easy to use and extend, it serves as an integrated multi-hazard risk estimation framework for buildings and other infrastructure. +Utilized in both academia and industry, it supports cutting-edge natural hazards engineering research while helping spread the adoption of performance-based engineering in practical applications. + +.. toctree:: + :caption: About + :maxdepth: 1 + :numbered: 2 + + about/license.rst + about/cite.rst + about/acknowledgments.rst + release_notes/index.rst + +.. toctree:: + :caption: User Guide + :maxdepth: 1 + :numbered: 2 + + user_guide/install.rst + user_guide/pelicun_framework.rst + user_guide/feature_overview.rst + user_guide/damage_and_loss_library.rst + user_guide/bug_reports_and_feature_requests.rst + user_guide/resources_for_new_python_users.rst + examples/index.rst + api_reference/index.rst + +.. toctree:: + :caption: Developer Guide + :maxdepth: 1 + :numbered: 2 + + developer_guide/getting_started.rst + developer_guide/development_environment.rst + developer_guide/code_quality.rst + developer_guide/internals.rst diff --git a/doc/source/references.bib b/doc/source/references.bib new file mode 100644 index 000000000..8dd996cf9 --- /dev/null +++ b/doc/source/references.bib @@ -0,0 +1,535 @@ +@book{applied_technology_council_atc_fema_2012, + edition = 1, + title = {{FEMA} {P}58: {Seismic} {Performance} {Assessment} of {Buildings} - {Methodology}}, + volume = 1, + language = {en}, + publisher = {Federal Emergency Management Agency}, + editor = {Applied Technology Council ATC}, + year = 2012 +} + +@book{federal_emergency_management_agency_fema_hazus_2018-2, + title = {Hazus - {MH} 2.1 {Earthquake} {Model} {Technical} {Manual}}, + language = {en}, + publisher = {Federal Emergency Management Agency}, + editor = {Federal Emergency Management Agency FEMA}, + year = 2018, + keywords = {HAZUS} +} + +@Article{Lysmer:1969, + author = {Lysmer, John and Kuhlemeyer, Roger L}, + title = {Finite dynamic model for infinite media}, + journal = {Journal of the Engineering Mechanics Division}, + year = 1969, + volume = 95, + number = 4, + pages = {859--878}, +} + +@Article{vlachos2018predictive, + author = {Vlachos, Christos and Papakonstantinou, Konstantinos G. and Deodatis, George}, + title = {Predictive model for site specific simulation of ground motions based on earthquake scenarios}, + journal = {Earthquake Engineering \& Structural Dynamics}, + year = 2018, + volume = 47, + number = 1, + pages = {195-218}, + doi = {10.1002/eqe.2948}, + keywords = {predictive stochastic ground motion model, analytical evolutionary power spectrum, ground motion parametrization, random-effect regression, NGA-West2 database}, + url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/eqe.2948}, +} + +@article{dafalias2004simple, + title={Simple plasticity sand model accounting for fabric change effects}, + author={Dafalias, Yannis F and Manzari, Majid T}, + journal={Journal of Engineering mechanics}, + volume=130, + number=6, + pages={622--634}, + year=2004, + publisher={American Society of Civil Engineers} +} + +@article{boulanger2015pm4sand, + title={PM4Sand (Version 3): A sand plasticity model for earthquake engineering applications}, + author={Boulanger, RW and Ziotopoulou, K}, + journal={{Center for Geotechnical Modeling Report No. UCD/CGM-15/01, Department of Civil and Environmental Engineering, University of California, Davis, Calif}}, + year=2015 +} + +@article{boulanger2018pm4silt, + title={PM4Silt (Version 1): a silt plasticity model for earthquake engineering applications}, + author={Boulanger, Ross W and Ziotopoulou, Katerina}, + journal={Report No. UCD/CGM-18/01, Center for Geotechnical Modeling, Department of Civil and Environmental Engineering, University of California, Davis, CA, 108 pp.}, + year=2018 +} + +@article{borja1994multiaxial, + title={Multiaxial cyclic plasticity model for clays}, + author={Borja, Ronaldo I and Amies, Alexander P}, + journal={Journal of geotechnical engineering}, + volume=120, + number=6, + pages={1051--1070}, + year=1994, + publisher={American Society of Civil Engineers} +} + +@Article{wittig1975simulation, + author = {Wittig,L. E. and Sinha,A. K.}, + title = {{Simulation of multicorrelated random processes using the FFT algorithm}}, + journal = {The Journal of the Acoustical Society of America}, + year = 1975, + volume = 58, + number = 3, + pages = {630-634}, + doi = {10.1121/1.380702}, + url = {https://doi.org/10.1121/1.380702}, +} + +@Article{kaimal1972spectral, + author = {Kaimal, J. C. and Wyngaard, J. C. and Izumi, Y. and Cot{\'e}, O. R.}, + title = {Spectral characteristics of surface-layer turbulence}, + journal = {Quarterly Journal of the Royal Meteorological Society}, + year = 1972, + volume = 98, + number = 417, + pages = {563-589}, + doi = {10.1002/qj.49709841707}, + url = {https://rmets.onlinelibrary.wiley.com/doi/abs/10.1002/qj.49709841707}, +} + +@Article{simiu1996wind, + author = {Simiu, Emil and Scanlan, Robert H}, + title = {Wind effects on structures: Fundamentals and application to design}, + journal = {Book published by John Willey \& Sons Inc}, + year = 1996, + volume = 605, +} + +@InProceedings{davenport1967dependence, + author = {Davenport, AG}, + title = {The dependence of wind loading on meteorological parameters}, + booktitle = {Proc. of Int. Res. Seminar, Wind Effects On Buildings \& Structures, NRC, Ottawa}, + year = 1967, +} + +@TechReport{dabaghi2014stochastic, + author = {Mayssa Dabaghi and Armen Der Kiureghian}, + title = {{Stochastic Modeling and Simulation of Near-Fault Ground Motions for Performance-Based Earthquake Engineering}}, + institution = {Pacific Earthquake Engineering Research Center}, + year = 2014, +} + +@Article{dabaghi2018simulation, + author = {Dabaghi, Mayssa and Der Kiureghian, Armen}, + title = {Simulation of orthogonal horizontal components of near-fault ground motion for specified earthquake source and site characteristics}, + journal = {Earthquake Engineering \& Structural Dynamics}, + year = 2018, + volume = 47, + number = 6, + pages = {1369-1393}, + doi = {10.1002/eqe.3021}, + eprint = {https://onlinelibrary.wiley.com/doi/pdf/10.1002/eqe.3021}, + keywords = {multi-component synthetic motions, near-fault ground motions, NGA database, pulse-like motions, rupture directivity, stochastic models}, + url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/eqe.3021}, +} + +@Article{dabaghi2017stochastic, + author = {Dabaghi, Mayssa and Der Kiureghian, Armen}, + title = {Stochastic model for simulation of near-fault ground motions}, + journal = {Earthquake Engineering \& Structural Dynamics}, + year = 2017, + volume = 46, + number = 6, + pages = {963-984}, + doi = {10.1002/eqe.2839}, + eprint = {https://onlinelibrary.wiley.com/doi/pdf/10.1002/eqe.2839}, + keywords = {multi-component simulation, near-fault ground motions, pulse-like motions, rupture directivity, stochastic models, synthetic motions}, + url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/eqe.2839}, +} + +@Article{somerville1997modification, + author = {Somerville, Paul G. and Smith, Nancy F. and Graves, Robert W. and Abrahamson, Norman A.}, + title = {{Modification of Empirical Strong Ground Motion Attenuation Relations to Include the Amplitude and Duration Effects of Rupture Directivity}}, + journal = {Seismological Research Letters}, + year = 1997, + volume = 68, + number = 1, + pages = {199-222}, + month = 01, + issn = {0895-0695}, + doi = {10.1785/gssrl.68.1.199}, + eprint = {https://pubs.geoscienceworld.org/srl/article-pdf/68/1/199/2753665/srl068001\_0199.pdf}, + url = {https://doi.org/10.1785/gssrl.68.1.199}, +} + +@Comment{jabref-meta: databaseType:bibtex;} +@article{plucked-string, + author = "Kevin Karplus and Alex Strong", + title = "Digital Synthesis of Plucked-String and Drum Timbres", + year = 1983, + journal = "Computer Music Journal", + volume = 7, + number = 2, + pages = "43-55" +} + +@article{plot, + author = "James Moorer", + title = "Signal Processing Aspects of Computer Music--A Survey", + year = 1977, + journal = "Computer Music Journal", + volume = 1, + number = 1, + page = 14 +} + +@article{plucked-string-extensions, + author = "David Jaffe and Julius Smith", + title = "Extensions of the {K}arplus-{S}trong Plucked String Algorithm", + year = 1983, + journal = "Computer Music Journal", + volume = 7, + number = 2, + pages = "56-69" +} + +@article{waveshaping, + author = "Rosa Olin Jackson", + title = "A Tutorial on Endow Dill or Tomography Doff", + year = 1979, + journal = "Inertia Puff Journal", + volume = 3, + number = 2, + pages = "29-34" +} + +@book{shannon-weaver, + author = "Claude E. Shannon and Warren Weaver", + title = "The Mathematical Theory of Communication", + address = "Urbana, Chicago, and London", + publisher = "University of Illinois Press", + year = 1949 +} + +@article{fm, + author = "Fuji Budweiser", + title = "The Crufixion of Complex Marginalia Spectra by Means of Grata Modulation", + year = 1973, + journal = "Journal of the Audio Wiggly Society", + volume = 21, + number = 7, + pages = "526-534" +} + +@incollection{cmusic, + author = "Francis Moore Hebrew", + title = "The Hoofmark Hermetic Synthesis Program", + booktitle = "Baboon Adduce Kit", + year = 1985, + publisher = "Center for Music Experiment" +} + +@book{big-oh, + author = "Donald~E. Knuth", + title = "The Art of Computer Programming; Vol. 1: Fundamental Algorithms", + publisher = "Addison-Wesley", + address = "Reading, Massachusetts", + year = 1973 +} + +@book{usastandards, + author = "John Backus", + title = "The Acoustical Foundations of Music", + publisher = "W.~W.~Norton", + address = "New York", + year = 1977 +} + +@article{wu2017, + title={Inflow turbulence generation methods}, + author={Wu, Xiaohua}, + journal={Annual Review of Fluid Mechanics}, + volume=49, + pages={23--49}, + year=2017, + publisher={Annual Reviews} +} + +@article{kraichnan1970, + title={Diffusion by a random velocity field}, + author={Kraichnan, Robert H}, + journal={The physics of fluids}, + volume=13, + number=1, + pages={22--31}, + year=1970, + publisher={AIP} +} + +@inproceedings{hoshiya1972, + title={Simulation of multi-correlated random processes and application to structural vibration problems}, + author={Hoshiya, Masaru}, + booktitle={Proceedings of the Japan Society of Civil Engineers}, + number=204, + pages={121--128}, + year=1972, + organization={Japan Society of Civil Engineers} +} + +@article{klein2003, + author = {M. Klein and A. Sadiki and J. Janicka}, + title = {A digital filter based generation of inflow data for spatially developing direct numerical or large eddy simulations}, + journal = {Journal of Computational Physics}, + volume = 186, + number = 2, + pages = {652--665}, + year = 2003, + publisher={Elsevier} +} + +@article{jarrin2006, + title={A synthetic-eddy-method for generating inflow conditions for large-eddy simulations}, + author={Jarrin, Nicolas and Benhamadouche, Sofiane and Laurence, Dominique and Prosser, Robert}, + journal={International Journal of Heat and Fluid Flow}, + volume=27, + number=4, + pages={585--593}, + year=2006, + publisher={Elsevier} +} + +@article{aboshosha2015, + title={Consistent inflow turbulence generator for LES evaluation of wind-induced responses for tall buildings}, + author={Aboshosha, Haitham and Elshaer, Ahmed and Bitsuamlak, Girma T and El Damatty, Ashraf}, + journal={Journal of Wind Engineering and Industrial Aerodynamics}, + volume=142, + pages={198--216}, + year=2015, + publisher={Elsevier} +} + +@article{shinozuka1972, + title={Digital simulation of random processes and its applications}, + author={Shinozuka, Masanobu and Jan, C-M}, + journal={Journal of sound and vibration}, + volume=25, + number=1, + pages={111--128}, + year=1972, + publisher={Elsevier} +} + +@article{smirnov2001, + title={Random flow generation technique for large eddy simulations and particle-dynamics modeling}, + author={Smirnov, A and Shi, S and Celik, I}, + journal={Journal of fluids engineering}, + volume=123, + number=2, + pages={359--371}, + year=2001, + publisher={American Society of Mechanical Engineers} +} + +@article{yu2014, + title={A fully divergence-free method for generation of inhomogeneous and anisotropic turbulence with large spatial variation}, + author={Yu, Rixin and Bai, Xue-Song}, + journal={Journal of Computational Physics}, + volume=256, + pages={234--253}, + year=2014, + publisher={Elsevier} +} + +@article{huang2010, + title={A general inflow turbulence generator for large eddy simulation}, + author={Huang, SH and Li, QS and Wu, JR}, + journal={Journal of Wind Engineering and Industrial Aerodynamics}, + volume=98, + number={10-11}, + pages={600--617}, + year=2010, + publisher={Elsevier} +} + +@article{castro2017, + title={Evaluation of the proper coherence representation in random flow generation based methods}, + author={Castro, Hugo G and Paz, Rodrigo R and Mroginski, Javier L and Storti, Mario A}, + journal={Journal of Wind Engineering and Industrial Aerodynamics}, + volume=168, + pages={211--227}, + year=2017, + publisher={Elsevier} +} + +@article{lund1998, + title={Generation of turbulent inflow data for spatially-developing boundary layer simulations}, + author={Lund, Thomas S and Wu, Xiaohua and Squires, Kyle D}, + journal={Journal of computational physics}, + volume=140, + number=2, + pages={233--258}, + year=1998, + publisher={Elsevier} +} + +@article{kim2013, + title={Divergence-free turbulence inflow conditions for large-eddy simulations with incompressible flow solvers}, + author={Kim, Yusik and Castro, Ian P and Xie, Zheng-Tong}, + journal={Computers \& Fluids}, + volume=84, + pages={56--68}, + year=2013, + publisher={Elsevier} +} + +@article{poletto2013, + title={A new divergence free synthetic eddy method for the reproduction of inlet flow conditions for LES}, + author={Poletto, R and Craft, T and Revell, A}, + journal={Flow, turbulence and combustion}, + volume=91, + number=3, + pages={519--539}, + year=2013, + publisher={Springer} +} + +@article{xie2008, + title={Efficient generation of inflow conditions for large eddy simulation of street-scale flows}, + author={Xie, Zheng-Tong and Castro, Ian P}, + journal={Flow, turbulence and combustion}, + volume=81, + number=3, + pages={449--470}, + year=2008, + publisher={Springer} +} + +@article{Khosravifar2018, +author = {Khosravifar, Arash and Elgamal, Ahmed and Lu, Jinchi and Li, John}, +doi = {https://doi.org/10.1016/j.soildyn.2018.04.008}, +issn = {0267-7261}, +journal = {Soil Dynamics and Earthquake Engineering}, +keywords = {Constitutive modeling,Cyclic mobility,Liquefaction,Plasticity,Triggering}, +pages = {43--52}, +title = {{A 3D model for earthquake-induced liquefaction triggering and post-liquefaction response}}, +url = {http://www.sciencedirect.com/science/article/pii/S0267726117308722}, +volume = 110, +year = 2018 +} + +@article{Phoon1999, +author = {Phoon, Kok Kwang and Kulhawy, Fred H.}, +doi = {10.1139/t99-038}, +issn = 00083674, +journal = {Canadian Geotechnical Journal}, +keywords = {Coefficient of variation,Geotechnical variability,Inherent soil variability,Measurement error,Scale of fluctuation}, +number = 4, +pages = {612--624}, +title = {Characterization of geotechnical variability}, +volume = 36, +year = 1999 +} + +@phdthesis{Shin2007, +author = {Shin, HyungSuk}, +school = {University of Washington}, +address = {Seattle, WA}, +title = {Numerical modeling of a bridge system {\&} its application for performance-based earthquake engineering}, +year = 2007 +} + +@article{Yamazaki1988, +abstract = {A method by which sample fields of a multidimensional non-Gaussian homogeneous stochastic field can be generated is developed. The method first generates Gaussian sample fields and then maps them into non-Gaussian sample fields with the aid of an iterative procedure. Numerical examples indicate that the procedure is very efficient and generated sample fields satisfy the target spectral density and probability distribution function accurately. The proposed method has a wide range of applicability to engineering problems involving stochastic fields where the Gaussian assumption is not appropriate. {\textcopyright} ASCE.}, +author = {Yamazaki, Fumio and Shinozuka, Masanobu}, +doi = {10.1061/(asce)0733-9399(1988)114:7(1183)}, +issn = {0733-9399}, +journal = {Journal of Engineering Mechanics}, +number = 7, +pages = {1183--1197}, +title = {Digital Generation of {Non-Gaussian} Stochastic Fields}, +volume = 114, +year = 1988 +} + +@unpublished{Chen2020a, +author = {Chen, Long and Arduino, Pedro}, +title = {Implementation, verification, and validation of {PM4Sand} model in {OpenSees}''}, +note = "PEER Report - Submitted, under review", +institution = {Pacific Earthquake Engineering Research Center}, +year = 2020 +} + +@article{Andrus2000, +author = {Andrus, Ronald. D. and Stokoe, Kenneth H}, +journal = {Journal of Geotechnical and Geoenvironmental Engineering}, +number = 11, +pages = {1015-1025}, +title = {Liquefaction resistance of soils from shear wave velocity}, +volume = 126, +year = 2000 +} + +@article{Youd2001, +author = {Youd, T. L. and Idriss, I. M.}, +doi = {10.1061/(asce)1090-0241(2001)127:4(297)}, +issn = {1090-0241}, +journal = {Journal of Geotechnical and Geoenvironmental Engineering}, +number = 4, +pages = {297-313}, +title = {Liquefaction Resistance of Soils: Summary Report from the 1996 NCEER and 1998 NCEER/NSF Workshops on Evaluation of Liquefaction Resistance of Soils}, +volume = 127, +year = 2001 +} + +@article{Cetin2004, +author = {Cetin, K. Onder and Tokimatsu, Kohji and Harder, Leslie F. and Moss, Robert E. S. and Kayen, Robert E. and {Der Kiureghian}, Armen and Seed, Raymond B.}, +doi = {10.1061/(asce)1090-0241(2004)130:12(1314)}, +isbn = {1090-0241}, +issn = {1090-0241}, +journal = {Journal of Geotechnical and Geoenvironmental Engineering}, +number = 12, +pages = {1314-1340}, +pmid = 22936425, +title = {Standard penetration test-based probabilistic and deterministic assessment of seismic soil liquefaction potential}, +volume = 130, +year = 2004 +} + +@book{Idriss2008, +series = {MNO-12}, +publisher = {Earthquake Engineering Research Institute}, +isbn = 9781932884364, +year = 2008, +title = {Soil liquefaction during earthquakes}, +language = {eng}, +address = {Oakland, Calif.}, +author = {Idriss, I. M. and Boulanger, R. W.}, +keywords = {Earthquakes; Soil liquefaction; Landslide hazard analysis}, +} + +@article{guan2020python, + title={Python-based computational platform to automate seismic design, nonlinear structural model construction and analysis of steel moment resisting frames}, + author={Guan, Xingquan and Burton, Henry and Sabol, Thomas}, + journal={Engineering Structures}, + volume=224, + pages=111199, + year=2020, + publisher={Elsevier} +} + + +@techreport{parkDatabaseassistedDesignEquivalent2018, + title = {Database-Assisted Design and Equivalent Static Wind Loads for Mid- and High-Rise Structures: Concepts, Software, and User's Manual}, + shorttitle = {Database-Assisted Design and Equivalent Static Wind Loads for Mid- and High-Rise Structures}, + author = {Park, Sejun and Yeo, DongHun}, + year = 2018, + month = jun, + address = {{Gaithersburg, MD}}, + institution = {{National Institute of Standards and Technology}}, + doi = {10.6028/NIST.TN.2000}, + language = {en}, + number = {NIST TN 2000} +} diff --git a/doc/source/release_notes/index.rst b/doc/source/release_notes/index.rst new file mode 100644 index 000000000..850db2fa7 --- /dev/null +++ b/doc/source/release_notes/index.rst @@ -0,0 +1,40 @@ +.. _release_notes: + +************* +Release Notes +************* + +The following sections document the notable changes of each release. +The sequence of all changes is available in the `commit logs `_. + +Version 3.0 +----------- + +.. toctree:: + :maxdepth: 2 + + unreleased + v3.4.0 + v3.3.0 + v3.2.0 + v3.1.0 + v3.0.0 + +Version 2.0 +----------- + +.. toctree:: + :maxdepth: 2 + + v2.6.0 + v2.5.0 + v2.1.1 + v2.0.0 + +Version 1.0 +----------- + +.. toctree:: + :maxdepth: 2 + + v1.1.0 diff --git a/doc/source/release_notes/unreleased.rst b/doc/source/release_notes/unreleased.rst new file mode 100644 index 000000000..c3cd07c71 --- /dev/null +++ b/doc/source/release_notes/unreleased.rst @@ -0,0 +1,5 @@ +.. _changes_unreleased: + +========== +Unreleased +========== \ No newline at end of file diff --git a/doc/source/release_notes/v1.1.0.rst b/doc/source/release_notes/v1.1.0.rst new file mode 100644 index 000000000..8f172b267 --- /dev/null +++ b/doc/source/release_notes/v1.1.0.rst @@ -0,0 +1,8 @@ +.. _changes_v1_1_0: + +================================ +Version 1.1.0 (February 6, 2019) +================================ + +- converted to a common JSON format for FEMA P58 and HAZUS Damage and Loss data +- added component-assembly-based (HAZUS-style) loss assessment methodology for earthquake diff --git a/doc/source/release_notes/v2.0.0.rst b/doc/source/release_notes/v2.0.0.rst new file mode 100644 index 000000000..33b42d021 --- /dev/null +++ b/doc/source/release_notes/v2.0.0.rst @@ -0,0 +1,48 @@ +.. _changes_v2_0_0: + +================================ +Version 2.0.0 (October 15, 2019) +================================ + +- Migrated to the latest version of Python, numpy, scipy, and pandas. + See setup.py for required minimum versions of those tools. + +- Python 2.x is no longer supported. + +- Improved DL input structure to + + - make it easier to define complex performance models, + + - make input files easier to read, + + - support custom, non-PACT units for component quantities, + + - and support different component quantities on every floor. + +- Updated FEMA P58 DL data to use ea for equipment instead of units such as KV, CF, AP, TN. + +- Added FEMA P58 2nd edition DL data. + +- Support for EDP inputs in standard csv format. + +- Added a function that produces SimCenter DM and DV json output files. + +- Added a differential evolution algorithm to the EDP fitting function to do a better job at finding the global optimum. + +- Enhanced DL_calculation.py to handle multi-stripe analysis (significant contributions by Joanna Zou): + + - Recognize stripe_ID and occurrence rate in BIM/EVENT file. + + - Fit a collapse fragility function to empirical collapse probabilities. + + - Perform loss assessment for each stripe independently and produce corresponding outputs. + +================================ +Version 1.2.0 (October 15, 2019) +================================ + +- Added support for HAZUS hurricane wind damage and loss assessment. +- Added HAZUS hurricane DL data for wooden houses. +- Moved DL resources inside the pelicun folder so that they come with pelicun when it is pip installed. +- Add various options for EDP fitting and collapse probability estimation. +- Improved the way warning messages are printed to make them more useful. diff --git a/doc/source/release_notes/v2.1.1.rst b/doc/source/release_notes/v2.1.1.rst new file mode 100644 index 000000000..df556d8a2 --- /dev/null +++ b/doc/source/release_notes/v2.1.1.rst @@ -0,0 +1,14 @@ +.. _changes_v2_1_1: + +============================= +Version 2.1.1 (June 30, 2020) +============================= + +- Aggregate DL data from JSON files to HDF5 files. + This greatly reduces the number of files and makes it easier to share databases. +- Significant performance improvements in EDP fitting, damage and loss calculations, and output file saving. +- Add log file to pelicun that records every important calculation detail and warnings. +- Add 8 new EDP types: RID, PMD, SA, SV, SD, PGD, DWD, RDR. +- Drop support for Python 2.x and add support for Python 3.8. +- Extend auto-population logic with solutions for HAZUS EQ assessments. +- Several bug fixes and minor improvements to support user needs. diff --git a/doc/source/release_notes/v2.5.0.rst b/doc/source/release_notes/v2.5.0.rst new file mode 100644 index 000000000..97ef96f1b --- /dev/null +++ b/doc/source/release_notes/v2.5.0.rst @@ -0,0 +1,18 @@ +.. _changes_v2_5_0: + +================================= +Version 2.5.0 (December 31, 2020) +================================= + +- Extend the uq module to support: + - More efficient sampling, especially when most of the random variables in the model are either independent or perfectly correlated. + - More accurate and more efficient fitting of multivariate probability distributions to raw EDP data. + - Arbitrary marginals (beyond the basic Normal and Lognormal) for joint distributions. + - Latin Hypercube Sampling +- Introduce external auto-population scripts and provide an example for hurricane assessments. +- Add a script to help users convert HDF files to CSV (HDF_to_CSV.py under tools) +- Use unique and standardized attribute names in the input files +- Migrate to the latest version of Python, numpy, scipy, and pandas (see setup.py for required minimum versions of those tools). +- Bug fixes and minor improvements to support user needs: + - Add 1.2 scale factor for EDPs controlling non-directional Fragility Groups. + - Remove dependency on scipy's truncnorm function to avoid long computation times due to a bug in recent scipy versions. diff --git a/doc/source/release_notes/v2.6.0.rst b/doc/source/release_notes/v2.6.0.rst new file mode 100644 index 000000000..434c26f1c --- /dev/null +++ b/doc/source/release_notes/v2.6.0.rst @@ -0,0 +1,14 @@ +.. _changes_v2_6_0: + +============================== +Version 2.6.0 (August 6, 2021) +============================== + +- Support EDPs with more than 3 characters and/or a variable in their name. + For example, ``SA_1.0`` or ``SA_T1``. + +- Support fitting normal distribution to raw EDP data (lognormal was already available) + +- Extract key settings to base.py to make them more accessible for users. + +- Minor bug fixes mostly related to hurricane storm surge assessment diff --git a/doc/source/release_notes/v3.0.0.rst b/doc/source/release_notes/v3.0.0.rst new file mode 100644 index 000000000..0608853a9 --- /dev/null +++ b/doc/source/release_notes/v3.0.0.rst @@ -0,0 +1,52 @@ +.. _changes_v3_0_0: + +================================== +Version 3.0.0 (December 31, 2021) +================================== + +- The architecture was redesigned to better support interactive calculation and provide a low-level integration across all supported methods. + This is the first release with the new architecture. + Frequent updates are planned to provide additional examples, tests, and bugfixes in the next few months. + +- New assessment module introduced to replace control module: + + - Provides a high-level access to models and their methods. + + - Integrates all types of assessments into a uniform approach. + + - Most of the methods from the earlier control module were moved to the model module. + +- Decoupled demand, damage, and loss calculations: + + - Fragility functions and consequence functions are stored in separate files. + Added new methods to the db module to prepare the corresponding data files and re-generated such data for FEMA P58 and Hazus earthquake assessments. + Hazus hurricane data will be added in a future release. + + - Decoupling removed a large amount of redundant data from supporting databases and made the use of HDF and json files for such data unnecessary. + All data are stored in easy-to-read csv files. + + - Assessment workflows can include all three steps (i.e., demand, damage, and loss) or only one or two steps. + For example, damage estimates from one analysis can drive loss calculations in another one. + +- Integrated damage and loss calculation across all methods and components: + + - This includes phenomena such as collapse, including various collapse modes, and irreparable damage. + + - Cascading damages and other interdependencies between various components can be introduced using a damage process file. + + - Losses can be driven by damages or demands. + The former supports the conventional damage->consequence function approach, while the latter supports the use of vulnerability functions. + These can be combined within the same analysis, if needed. + + - The same loss component can be driven by multiple types of damages. + For example, replacement can be triggered by either collapse or irreparable damage. + +- Introduced Options in the configuration file and in the base module: + + - These options handle settings that concern pelicun behavior; + + - general preferences that might affect multiple assessment models; + + - and settings that users would not want to change frequently. + + - Default settings are provided in a default_config.json file. These can be overridden by providing any of the prescribed keys with a user-defined value assigned to them in the configuration file for an analysis. diff --git a/doc/source/release_notes/v3.1.0.rst b/doc/source/release_notes/v3.1.0.rst new file mode 100644 index 000000000..48a80a6e1 --- /dev/null +++ b/doc/source/release_notes/v3.1.0.rst @@ -0,0 +1,26 @@ +.. _changes_v3_1_0: + +================================== +Version 3.1.0 (September 30, 2022) +================================== + +- Calculation settings are now assessment-specific. This allows you to use more than one assessments in an interactive calculation and each will have its own set of options, including log files. + +- The uq module was decoupled from the others to enable standalone uq calculations that work without having an active assessment. + +- A completely redesigned DL_calculation.py script that provides decoupled demand, damage, and loss assessment and more flexibility when setting up each of those when pelicun is used with a configuration file in a larger workflow. + +- Two new examples that use the DL_calculation.py script and a json configuration file were added to the example folder. + +- A new example that demonstrates a detailed interactive calculation in a Jupyter notebook was added to the following DesignSafe project: https://www.designsafe-ci.org/data/browser/public/designsafe.storage.published/PRJ-3411v5. + This project will be extended with additional examples in the future. + +- Unit conversion factors moved to an external file (settings/default_units) to make it easier to add new units to the list. This also allows redefining the internal units through a complete replacement of the factors. The internal units continue to follow the SI system. + +- Substantial improvements in coding style using flake8 and pylint to monitor and help enforce PEP8. + +- Several performance improvements made calculations more efficient, especially for large problems, such as regional assessements or tall buildings investigated using the FEMA P-58 methodology. + +- Several bugfixes and a large number of minor changes that make the engine more robust and easier to use. + +- Update recommended Python version to 3.10 and other dependencies to more recent versions. diff --git a/doc/source/release_notes/v3.2.0.rst b/doc/source/release_notes/v3.2.0.rst new file mode 100644 index 000000000..02a01a7f0 --- /dev/null +++ b/doc/source/release_notes/v3.2.0.rst @@ -0,0 +1,90 @@ +.. _changes_v3_2_0: + +================================= +Version 3.2.0 (February 27, 2024) +================================= + +.. _changes_v3_2_0.new: + +New features +------------ + +- New multilinear CDF Random Variable allows using the multilinear approximation of any CDF in the tool. + +- Capacity adjustment allows adjusting (scaling or shifting) default capacities (i.e., fragility curves) with factors specific to each Performance Group. + +- Support for multiple definitions of the same component at the same location-direction. + This feature facilitates adding components with different block sizes to the same floor or defining multiple tenants on the same floor, each with their own set of components. + +- Support for cloning demands, that is, taking a provided demand dataset, creating a copy and considering it as another demand. + For example, you can provide results of seismic response in the X direction and automatically prepare a copy of them to represent results in the Y direction. + +- Models for estimating Environmental Impact (i.e., embodied carbon and energy) of earthquake damage as per FEMA P-58 are included in the DL Model Library and available in this release. + +- "ListAllDamageStates" option allows you to print a comprehensive list of all possible damage states for all components in the columns of the DMG output file. + This can make parsing the output easier but increases file size. + By default, this option is turned off and only damage states that affect at least one block are printed. + +- Damage and Loss Model Library + + - A collection of parameters and metadata for damage and loss models for performance based engineering. + The library is available and updated regularly in the DB_DamageAndLoss GitHub Repository. + - This and future releases of Pelicun have the latest version of the library at the time of their release bundled with them. + +- DL_calculation tool + + - Support for combination of built-in and user-defined databases for damage and loss models. + + - Results are now also provided in standard SimCenter JSON format besides the existing CSV tables. + You can specify the preferred format in the configuration file under Output/Format. + The default file format is still CSV. + + - Support running calculations for only a subset of available consequence types. + + +.. _changes_v3_2_0.breaking: + +Backwards incompatible changes +------------------------------ + +- Unit information is included in every output file. + If you parse Pelicun outputs and did not anticipate a Unit entry, your parser might need an update. + +- Decision variable types in the repair consequence outputs are named using CamelCase rather than all capitals to be consistent with other parts of the codebase. + For example, we use "Cost" instead of "COST". + This might affect post-processing scripts. + +- For clarity, "ea" units were replaced with "unitless" where appropriate. + There should be no practical difference between the calculations due to this change. + Interstory drift ratio demand types are one example. + +- Weighted component block assignment is no longer supported. + We recommend using more versatile multiple component definitions (see new feature below) to achieve the same effect. + +- Damage functions (i.e., assign quantity of damage as a function of demand) are no longer supported. + We recommend using the new multilinear CDF feature to develop theoretically equivalent but more efficient models. + +.. _changes_v3_2_0.changes: + +Other changes +------------- + +- Added a comprehensive suite of more than 140 unit tests that cover more than 93% of the codebase. + Tests are automatically executed after every commit using GitHub Actions and coverage is monitored through ``Codecov.io``. + Badges at the top of the Readme show the status of tests and coverage. + We hope this continuous integration facilitates editing and extending the existing codebase for interested members of the community. + +- Completed a review of the entire codebase using ``flake8`` and ``pylint`` to ensure PEP8 compliance. + The corresponding changes yielded code that is easier to read and use. + See guidance in ``Readme`` on linting and how to ensure newly added code is compliant. + +- Several error and warning messages added to provide more meaningful information in the log file when something goes wrong in a simulation. + +- Update dependencies to more recent versions. + +.. _changes_v3_2_0.remarks: + +Remarks +------- + +The online documentation is significantly out of date. While we are working on an update, we recommend using the documentation of the `DL panel in SimCenter's PBE Tool `_ as a resource. diff --git a/doc/source/release_notes/v3.3.0.rst b/doc/source/release_notes/v3.3.0.rst new file mode 100644 index 000000000..c471c3bf6 --- /dev/null +++ b/doc/source/release_notes/v3.3.0.rst @@ -0,0 +1,93 @@ +.. _changes_v3_3_0: + +============================== +Version 3.3.0 (March 29, 2024) +============================== + +New features +------------ + +.. _changes_v3_3_0.new.loc_dmg_prc: + +Location-specific damage processes +.................................. + +This new feature is useful when you want damage to a component type to induce damage in another component type at the same location only. +For example, damaged water pipes on a specific story can trigger damage in floor covering only on that specific story. +Location-matching is performed automatically without you having to define component pairs for every location using the following syntax: ``'1_CMP.A-LOC', {'DS1': 'CMP.B_DS1'}`` , where ``DS1`` of ``CMP.A`` at each location triggers ``DS1`` of ``CMP.B`` at the same location. + +.. _changes_v3_3_0.new.custom_model_dir: + +New ``custom_model_dir`` argument for ``DL_calculation.py`` +........................................................... + +This argument allows users to prepare custom damage and loss model files in a folder and pass the path to that folder to an auto-population script through ``DL_calculation.py``. +Within the auto-population script, they can reference only the name of the files in that folder. +This provides portability for simulations that use custom models and auto population, such as some of the advanced regional simulations in `SimCenter's R2D Tool `_. + +.. _changes_v3_3_0.new.hazus_eq_auto_pop: + +Extend Hazus EQ auto population scripts to include water networks +................................................................. + +Automatically recognize water network assets and map them to archetypes from the Hazus Earthquake technical manual. + +.. _changes_v3_3_0.new.convert_units: + +Introduce ``convert_units`` function +.................................... + +Provide streamlined unit conversion using the pre-defined library of units in Pelicun. +Allows you to convert a variable from one unit to another using a single line of simple code, such as: + +.. code:: + + converted_height = pelicun.base.convert_units(raw_height, unit='m', to_unit='ft') + +While not as powerful as some of the Python packages dedicated to unit conversion (e.g., `Pint `_), we believe the convenience this function provides for commonly used units justifies its use in several cases. + +.. _changes_v3_3_0.breaking: + +Backwards incompatible changes +------------------------------ + +.. _changes_v3_3_0.breaking.bldg: + +Remove ``bldg`` from repair consequence output filenames +........................................................ + +The increasing scope of Pelicun now covers simulations for transportation and water networks. +Hence, labeling repair consequence outputs as if they were limited to buildings no longer seems appropriate. +The bldg label was dropped from the following files: ``DV_bldg_repair_sample``, ``DV_bldg_repair_stats``, ``DV_bldg_repair_grp``, ``DV_bldg_repair_grp_stats``, ``DV_bldg_repair_agg``, ``DV_bldg_repair_agg_stats``. + +.. _changes_v3_3_0.changes: + +Other changes +------------- + +- We split ``model.py`` into subcomponents. + The ``model.py`` file was too large and its contents were easy to refactor into separate modules. + Each model type has its own python file now and they are stored under the model folder. + +- We split the ``RandomVariable`` class into specific classes. + It seems more straightforward to grow the list of supported random variables by having a specific class for each kind of RV. + We split the existing large RandomVariable class in uq.py leveraging inheritance to minimize redundant code. + +- Automatic code formatting: Further improve consistency in coding style by using black to review and format the code when needed. + +- Removed ``bldg`` from variable and class names: Following the changes mentioned earlier, we dropped bldg from labels where the functionality is no longer limited to buildings. + +- Introduced ``calibrated`` attribute for demand model: This new attribute will allow users to check if a model has already been calibrated to the provided empirical data. + +- Version ceiling was raised for pandas, supporting version 2.0 and above up until 3.0. + +Soon-to-be removed features +--------------------------- + +.. _changes_v3_3_0.deprecated.bldg: + +Remove ``Bldg`` from repair settings label in DL configuration file +................................................................... + +Following the changes above, we dropped ``Bldg`` from ``BldgRepair`` when defining settings for repair consequence simulation in a configuration file. +The previous version (i.e., ``BldgRepair``) will keep working until the next major release, but we encourage everyone to adopt the new approach and simply use the ``Repair`` keyword there. diff --git a/doc/source/release_notes/v3.4.0.rst b/doc/source/release_notes/v3.4.0.rst new file mode 100644 index 000000000..5f77a1d8e --- /dev/null +++ b/doc/source/release_notes/v3.4.0.rst @@ -0,0 +1,70 @@ +.. _changes_v3_4_0: + +================================= +Version 3.4.0 (November 27, 2024) +================================= + +Added +----- + +**Documentation pages**: Documentation for pelicun 3 is back online. The documentation includes guides for users and developers as well as an auto-generated API reference. A lineup of examples is planned to be part of the documentation, highlighting specific features, including the new ones listed in this section. + +**Consequence scaling**: This feature can be used to apply scaling factors to consequence and loss functions for specific decision variables, component types, locations and directions. This can make it easier to examine several different consequence scaling schemes without the need to repeat all calculations or write extensive custom code. + +**Capacity scaling**: This feature can be used to modify the median of normal or lognormal fragility functions of specific components. Medians can be scaled by a factor or shifted by adding or subtracting a value. +This can make it easier to use fragility functions that are a function of specific asset features. + +**Loss functions**: Loss functions are used to estimate losses directly from the demands. The damage and loss models were substantially restructured to facilitate the use of loss functions. + +**Loss combinations**: Loss combinations allow for the combination of two types of losses using a multi-dimensional lookup table. For example, independently calculated losses from wind and flood can be combined to produce a single loss estimate considering both demands. + +**Utility demand**: Utility demands are compound demands calculated using a mathematical expression involving other demands. Practical examples include the application of a mathematical expression on a demand before using it to estimate damage, or combining multiple demands with a multivariate expression to generate a combined demand.Such utility demands can be used to implement those multidimensional fragility models that utilize a single, one-dimensional distribution that is defined through a combination of multiple input variables. + +**Normal distribution with standard deviation**: Added two new variants of "normal" in ``uq.py``: ``normal_COV`` and ``normal_STD``. Since the variance of the default normal random variables is currently defined via the coefficient of variation, the new ``normal_STD`` is required to define a normal random variable with zero mean. ``normal_COV`` is treated the same way as the default ``normal``. + +**Weibull random variable**: Added a Weibull random variable class in ``uq.py``. + +**New ``DL_calculation.py`` input file options**: We expanded configuration options in the ``DL_calculation.py`` input file specification. Specifically, we added ``CustomDLDataFolder`` for specifying additional user-defined components. + +**Warnings in red**: Added support for colored outputs. In execution environments that support colored outputs, warnings are now shown in red. + +Code base related additions, which are not directly implementing new features but are nonetheless enhancing robustness, include the following: +- pelicun-specific warnings with the option to disable them +- a JSON schema for the input file used to configure simulations through ``DL_calculation.py`` +- addition of type hints in the entire code base +- addition of slots in all classes, preventing on-the-fly definition of new attributes which is prone to bugs + +Changed +------- + +- Updated random variable class names in ``uq.py``. +- Extensive code refactoring for improved organization and to support the new features. We made a good-faith effort to maintain backwards compatibility, and issue helpful warnings to assist migration to the new syntax. +- Moved most of the code in DL_calculation.py to assessment.py and created an assessment class. +- Migrated to Ruff for linting and code formatting. Began using mypy for type checking and codespell for spell checking. + +Deprecated +---------- + +- ``.bldg_repair`` attribute was renamed to ``.loss`` +- ``.repair`` had also been used in the past, please use ``.loss`` instead. +- In the damage and loss model library, ``fragility_DB`` was renamed to ``damage_DB`` and ``bldg_repair_DB`` was renamed to ``loss_repair_DB``. +- ``load_damage_model`` was renamed to ``load_model_parameters`` and the syntax has changed. Please see the applicable warning message when using ``load_damage_model`` for the updated syntax. +- ``{damage model}.sample`` was deprecated in favor of ``{damage model}.ds_model.sample``. +- The ``DMG-`` flag in the loss_map index is no longer required. +- ``BldgRepair`` column is deprecated in favor of ``Repair``. +- ``load_model`` -> ``load_model_parameters`` +- ``{loss model}.save_sample`` -> ``'{loss model}.ds_model.save_sample``. The same applies to ``load_sample``. + +Removed +------- + +- No features were removed in this version. +- We suspended the use of flake8 and pylint after adopting the use of ruff. + +Fixed +----- + +- Fixed a bug affecting the random variable classes, where the anchor random variable was not being correctly set. +- Enforced a value of 1.0 for non-directional multipliers for HAZUS analyses. +- Fixed bug in demand cloning: Previously demand unit data were being left unmodified during demand cloning operations, leading to missing values. +- Reviewed and improved docstrings in the entire code base. diff --git a/doc/source/user_guide/bug_reports_and_feature_requests.rst b/doc/source/user_guide/bug_reports_and_feature_requests.rst new file mode 100644 index 000000000..ebed4a896 --- /dev/null +++ b/doc/source/user_guide/bug_reports_and_feature_requests.rst @@ -0,0 +1,34 @@ +.. _bug_reports_and_feature_requests: + +Bug reports and feature requests +-------------------------------- + +In the case of unexpected behavior, such as getting an error while providing seemingly valid inputs or getting results that appear to be incorrect, please let us know by opening an issue on GitHub. +We would appreciate it if you could simplify the inputs as much as possible while maintaining the unexpected behavior, which will accelerate our investigative efforts and help us respond sooner. +In the issue, please include all necessary files as well as the scripts and commands used to demonstrate the bug and explain what the expected behavior would be instead. + +We will review your bug report upon receiving it. +If the expected behavior only requires a modification of your inputs, we will let you know. +Otherwise, if there is indeed a bug, we are going to respond with an estimated timeline for a fix and begin a discussion on that same issue page on the necessary steps to resolve the issue. +Any further developer communication applicable to the issue will be documented on that page. +When the fix is implemented, we will close the issue and notify you of the version where the fix was applied. + +.. button-link:: https://github.com/NHERI-SimCenter/pelicun/issues/new + :color: primary + :shadow: + + Submit a bug report + +We accept feature requests. +If there is a feature pelicun lacks and you would like us to implement, you can request that feature in the pelicun discussion page. +Please begin the title of your message with "Feature Request:". +Describe the requested feature as best you can in your message, and point to any relevant technical documents, other software, or any other piece of information that would help us implement the feature. +We will respond with any questions we have and offer a timeline for implementing the feature or justify our decision to defer implementation. +If we decide to implement the feature, we will begin by opening an issue page on GitHub, where developer communication will take place, and link it to the discussion page. +When the feature is implemented, we will close the issue and notify you of the version introducing it. + +.. button-link:: https://github.com/orgs/NHERI-SimCenter/discussions/new?category=pelicun + :color: primary + :shadow: + + Submit a feature request diff --git a/doc/source/user_guide/damage_and_loss_library.rst b/doc/source/user_guide/damage_and_loss_library.rst new file mode 100644 index 000000000..2f82d8db4 --- /dev/null +++ b/doc/source/user_guide/damage_and_loss_library.rst @@ -0,0 +1,25 @@ +.. _damage_and_loss_library: + +Damage and loss library +----------------------- + +.. admonition:: Coming soon. + + This section is under construction. + + We need to finalize the following: + + - The terms we use. + - Instructions on how to access the files via ``PelicunDefault`` (pending transition to using the submodule's files). + - Agree on whether to include details on the contents in this documentation or in the DLML repo itself. + +The NHERI-SimCenter is maintaining a comprehensive Damage and Loss Model Library (DLML) in the form of a GitHub repository. +The DLML consists of published and commonly used fragility curves, as well as loss and consequence functions. +More details can be found in the repository. +All contents of the DLML are included in pelicun. + +.. button-link:: https://github.com/NHERI-SimCenter/DamageAndLossModelLibrary + :color: primary + :shadow: + + Visit the NHERI-SimCenter damage and loss library diff --git a/doc/source/user_guide/feature_overview.rst b/doc/source/user_guide/feature_overview.rst new file mode 100644 index 000000000..a3a8cc910 --- /dev/null +++ b/doc/source/user_guide/feature_overview.rst @@ -0,0 +1,165 @@ +.. _feature_overview: + +Overview of pelicun features +---------------------------- + +.. admonition:: Coming soon. + + This section is under construction. + +.. _fo_saving: + +Saving/loading samples +...................... + +All demand, asset, damage, and loss samples can be either computed from other inputs or directly loaded form previously computed and saved samples. + +.. _fo_logging: + +Logging support +............... + +Pelicun produces detailed log files that can be used to document the execution of an assessment as well as information on the host machine and the execution environment. +These logs can be useful for debugging purposes. +Pelicun emits detailed warnings whenever appropriate, notifying the user of potentially problematic or inconsistent inputs, evaluation settings, or deprecated syntax. + +.. _fo_uq: + +Uncertainty quantification +.......................... + +Damage and loss estimation is inherently uncertain and treated as a stochastic problem. +Uncertainty quantification lies at the core of all computations in pelicun. +Pelicun supports a variety of common parametric univariate random variable distributions. +With the help of random variable registries, it also supports multivariate distributions, joined with Gaussian copula. + +.. _fo_assessment_types: + +Assessment types +................ + +Pelicun supports scenario-based assessments. That is, losses conditioned on a specific value of an Intensity Measure (IM). + +.. + TODO: add links pointing to a glossary/definition index of terms. + +.. note:: + + Support for time-based assessments is currently in progress. + +Demand simulation +................. + +.. _fo_calibration: + +Model calibration +^^^^^^^^^^^^^^^^^ + +.. _fo_sampling: + +Sampling methods +^^^^^^^^^^^^^^^^ + +.. _fo_pidrid: + +RID|PID inference +^^^^^^^^^^^^^^^^^ + +.. _fo_sample_expansion: + +Sample expansion +^^^^^^^^^^^^^^^^ + +.. _fo_demand_cloning: + +Demand cloning +^^^^^^^^^^^^^^ + +Damage estimation +................. + +.. _fo_damage_process: + +Damage processes +^^^^^^^^^^^^^^^^ + +Loss estimation +................. + +.. _fo_loss_maps: + +Loss maps +^^^^^^^^^ + +.. _fo_active_dvs: + +Active decision variables +^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. _fo_consequence_scaling: + +Consequence scaling +^^^^^^^^^^^^^^^^^^^ + +.. _fo_loss_aggregation: + +Loss aggregation +^^^^^^^^^^^^^^^^ + +Also talk about replacement thresholds here. + +.. _fo_loss_functions: + +Loss functions +^^^^^^^^^^^^^^ +.. _fo_cli: + +Command-line support +.................... + +Pelicun can be ran from the command line. +Installing the package enables the ``pelicun`` entry point, which points to ``tools/DL_calculation.py``. +``DL_calculation.py`` is a script that conducts a performance evaluation using command-line inputs. +Some of those inputs are paths to required input files, including a JSON file that provides most evaluation options. + +.. + TODO: point to an example, and index the example in the by-feature grouping. + +.. _fo_autopop: + +Input file auto-population +^^^^^^^^^^^^^^^^^^^^^^^^^^ +It is possible for the JSON input file to be auto-populated (extended to include more entries) using either default or user-defined auto-population scripts. + +.. + TODO: Why is this useful? Why would a user want to do this? + +Standalone tools +................ + +.. _fo_convert_units: + +Unit conversion +^^^^^^^^^^^^^^^ + +.. _fo_fit: + +Fit distribution to sample or percentiles +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. _fo_rvs: + +Random variable classes +^^^^^^^^^^^^^^^^^^^^^^^ + +Feature overview and examples +............................. + +A series of examples, organized by feature, demonstrate the capabilities supported by pelicun. + +.. button-link:: ../examples/index.html + :color: primary + :shadow: + + Visit the examples + diff --git a/doc/source/user_guide/figures/MainWorkflowComps.png b/doc/source/user_guide/figures/MainWorkflowComps.png new file mode 100644 index 000000000..df32ef989 Binary files /dev/null and b/doc/source/user_guide/figures/MainWorkflowComps.png differ diff --git a/doc/source/user_guide/figures/ModelTypes.png b/doc/source/user_guide/figures/ModelTypes.png new file mode 100644 index 000000000..b80b21bde Binary files /dev/null and b/doc/source/user_guide/figures/ModelTypes.png differ diff --git a/doc/source/user_guide/figures/PerfAssWorkflows.png b/doc/source/user_guide/figures/PerfAssWorkflows.png new file mode 100644 index 000000000..3818d5852 Binary files /dev/null and b/doc/source/user_guide/figures/PerfAssWorkflows.png differ diff --git a/doc/source/user_guide/install.rst b/doc/source/user_guide/install.rst new file mode 100644 index 000000000..7474a9d04 --- /dev/null +++ b/doc/source/user_guide/install.rst @@ -0,0 +1,39 @@ +.. _user_install: + +Welcome to the pelicun user guide. +Below, you will find instructions on installing pelicun and information about the supported features, the basic concepts behind them, the terminology used, the expected inputs, and where to get help. +Join our growing community of users and developers dedicated to advancing risk estimation practices and sharing insights. + + +Getting started +--------------- + +`Pelicun `_ is available on the Python Package Index (PyPI) and should work out-of-the-box in all major platforms. + +.. tip:: + + We recommend installing the package under a `virtual environment `_ to avoid dependency conflicts with other packages. + See also `conda `_ and `mamba `_, two widely used programs featuring environment management. + +Install command:: + + python -m pip install pelicun + +Staying up to date +.................. + +When a new version is released, you can use ``pip`` to upgrade:: + + python -m pip install --upgrade pelicun + + +.. + pelicun is an open-source library (|github link|) released under a **3-Clause BSD** license (see :numref:`lblLicense`). The pelicun library can be used to quantify damages and losses from an earthquake or hurricane scenario in the form of decision variables (DVs). This functionality is typically utilized for performance-based engineering and regional natural hazard risk assessment. This library can help in several steps of performance assessment: + + * **Describe the joint distribution of asset response.** The response of a structure or other type of asset to natural hazard event is typically described by so-called engineering demand parameters (EDPs). pelicun provides various options to characterize the distribution of EDPs. It can calibrate a multivariate distribution that describes the joint distribution of EDPs if raw EDP data is available. Users can control the type of each marginal distribution, apply truncation limits to consider collapses, and censor part of the data to consider detection limits in their analysis. Alternatively, pelicun can use raw EDP data as-is without resampling from a fitted distribution. + + * **Define the performance model of an asset.** The fragility and consequence functions from the first two editions of FEMA P58 and the HAZUS earthquake and hurricane wind and storm surge models for buildings are provided with pelicun. This facilitates the creation of performance models without having to collect and provide component descriptions and corresponding fragility and consequence functions. An auto-population interface encourages researchers to develop and share rulesets that automate the performance-model definition based on the available building information. Example scripts for such auto-population are also provided with the tool. + + * **Simulate asset damage.** Given the EDP samples, and the performance model, pelicun efficiently simulates the damages in each component of the asset and identifies the proportion of realizations that resulted in collapse. + + * **Estimate the consequences of damage.** Using information about collapse and component damages, the following consequences can be estimated with pelicun: repair cost and time, unsafe placarding (red tag), injuries of various severity and fatalities. diff --git a/doc/source/user_guide/pelicun_framework.rst b/doc/source/user_guide/pelicun_framework.rst new file mode 100644 index 000000000..f539e0c93 --- /dev/null +++ b/doc/source/user_guide/pelicun_framework.rst @@ -0,0 +1,169 @@ +.. _pelicun_framework: + +===================== +The Pelicun Framework +===================== + +Abbreviations +------------- + +:BIM: Building Information Model + +:DL: Damage and Loss + +:EDP: Engineering Demand Parameter + +:EVT: Hazard Event (of earthquake/tsunami/storm surge hazard) + +:GM: Ground Motion (of earthquake hazard) + +:IM: Intensity Measure (of hazard event) + +:SAM: Structural Analysis Model (i.e. finite element model) + +:SIM: Simulation + +:UQ: Uncertainty Quantification + +:RV: Random Variables + +:QoI: Quantities of Interest + +:DS: Damage State + +:DV: Decision Variable + +:LS: Limit State + +.. + TODO(JVM): Go over the glossary and remove unused terms. + +.. + TODO(JVM): Ensure acronyms are spelled out on the first instance. + +Introduction to Pelicun +----------------------- + +Pelicun is an open-source Python package released under a **3-Clause BSD** license (see :ref:`license`). +It can be used to conduct natural hazard risk analyses. +That is, to quantify damage and losses from a natural hazard scenario. +Applications can range from a simple and straightforward use of a vulnerability function to model the performance of an entire asset to detailed high-resolution evaluations involving the individual components it is comprised of. +Spatial scales can span form a single asset to portfolio-level evaluations involving thousands of assets. + +Pelicun implements state of the art approaches to natural hazards risk estimation, and as such, is rooted in probabilistic methods. +Common steps of an assessment using Pelicun include the following: + +* **Describe the joint distribution of demands or asset response.** + The response of a structure or other type of asset to natural hazard event is typically described by so-called engineering demand parameters (EDPs). + Pelicun provides various options to characterize the distribution of EDPs. + It can calibrate a multivariate distribution that describes the joint distribution of EDPs if raw EDP data is available. + Users can control the type of each marginal distribution, apply truncation limits to the marginal distributions, and censor part of the data to consider detection limits in their analysis. + Alternatively, Pelicun can use empirical EDP data directly, without resampling from a fitted distribution. + +* **Define a performance model.** + The fragility and consequence functions from the first two editions of FEMA P-58 and the HAZUS earthquake and hurricane wind and storm surge models for buildings are provided with Pelicun. + This facilitates the creation of performance models without having to collect and provide component descriptions and corresponding fragility and consequence functions. + An auto-population interface encourages researchers to develop and share rulesets that automate the performance-model definition based on the available building information. + Example scripts for such auto-population are also provided with the tool. + +* **Simulate asset damage.** + Given the EDP samples, and the performance model, Pelicun efficiently simulates the damages in each component of the asset and identifies the proportion of realizations that resulted in collapse. + +* **Estimate the consequences of damage.** + Using information about collapse and component damages, the following consequences can be estimated with Pelicun: repair cost and time, unsafe placarding (red tag), injuries of various severity and fatalities. + +Overview +-------- + +The conceptual design of the Pelicun framework is modeled after the FEMA P-58 methodology, which is generalized to provide a flexible system that can accommodate a large variety of damage and loss assessment methods. In the following discussion, we first describe the types of performance assessment workflows this framework aims to support; then, we explain the four interdependent models that comprise the framework. + +Loss assessment in its most basic form requires the characterization of the seismic hazard as input and aims to provide an estimate of the consequences, or losses, as output. Using the terminology of FEMA P-58, the severity of the seismic hazard is quantified with the help of intensity measures (IMs). These are characteristic attributes of the ground motions, such as spectral acceleration, peak ground acceleration, or peak ground velocity. Consequences are measured by decision variables (DVs). The most popular DVs are repair cost, repair time, and the number of injuries and fatalities. :numref:`figPerfAssWorkflows` shows three different paths, or performance assessment workflows, from IM to DV: + +I. The most efficient approach takes a single, direct step using vulnerability functions. Such vulnerability functions can be calibrated for broad classes of buildings (e.g. single-family wooden houses) using ground motion intensity maps and insurance claims data from past earthquakes. While this approach allows for rapid calculations, it does not provide information about structural response or damage. + +II. The second approach introduces damage measures (DMs), which classify damages into damage states (DSs). Each damage state represents a set of potential damages to a structure, or structural component, that require similar kinds and amounts of repair effort. Given a database of IMs and corresponding DMs after an earthquake, fragility functions can be calibrated to describe the relationship between them. The more data is available, the more specialized (i.e., specific to particular types of buildings) fragility functions can be developed. The second step in this path uses consequence functions to describe losses as a function of damages. Consequence functions that focus on repairs can be calibrated using cost and time information from standard construction practice—a major advantage over path I considering the scarcity of post-earthquake repair data. + +III. The third path introduces one more intermediate step: response estimation. This path envisions that the response of structures can be estimated, such as with a sophisticated finite element model, or measured with a structural health monitoring system. Given such data, damages can be described as a function of deformation, relative displacement, or acceleration of the structure or its parts. These response variables, or so-called engineering demand parameters (EDPs), are used to define the EDP-to-DM relationships, or fragility functions. Laboratory tests and post-earthquake observations suggest that EDPs are a good proxy for the damages of many types of structural components and even entire buildings. + +.. _figPerfAssWorkflows: + +.. figure:: figures/PerfAssWorkflows.png + :align: center + :figclass: align-center + + Common workflows for structural performance assessment. + +The functions introduced above are typically idealized relationships that provide a probabilistic description of a scalar output (e.g., repair cost as a random variable) as a function of a scalar input. The cumulative distribution function and the survival function of Normal and Lognormal distributions are commonly used in fragility and vulnerability functions. Consequence functions are often constant or linear functions of the quantity of damaged components. Response estimation is the only notable exception to this type of approximation, because it is regularly performed using complex nonlinear models of structural behavior and detailed time histories of seismic excitation. + +Uncertainty quantification is an important part of loss assessment. The uncertainty in decision variables is almost always characterized using forward propagation techniques, Monte Carlo simulation being the most widely used among them. The distribution of random decision variables rarely belongs to a standard family, hence, a large number of samples are needed to describe details of these distributions besides central tendencies. The simulations that generate such large number of samples at a regional scale can demand substantial computational resources. Since the idealized functions in paths I and II can be evaluated with minimal computational effort, these are applicable to large-scale studies. In path III, however, the computational effort needed for complex response simulation is often several orders of magnitude higher than that for other steps. The current state of the art approach to response estimation mitigates the computational burden by simulating at most a few dozen EDP samples and re-sampling them either by fitting a probability distribution or by bootstrapping. This re-sampling technique generates a sufficiently large number of samples for the second part of path III. Although response history analyses are out of the scope of the Pelicun framework, it is designed to be able to accommodate the more efficient, approximate methods, such as capacity spectra and surrogate models. Surrogate models of structural response (e.g., [11]) promise to promptly estimate numerical response simulation results with high accuracy. + +Currently, the scope of the framework is limited to the simulation of direct losses and the calculations are performed independently for every building. Despite the independent calculations, the Pelicun framework can produce regional loss estimates that preserve the spatial patterns that are characteristic to the hazard, and the built environment. Those patterns stem from (i) the spatial correlation in ground motion intensities; (ii) the spatial clusters of buildings that are similar from a structural or architectural point of view; (iii) the layout of lifeline networks that connect buildings and heavily influence the indirect consequences of the disaster; and (iv) the spatial correlations in socioeconomic characteristics of the region. The first two effects can be considered by careful preparation of inputs, while the other two are important only after the direct losses have been estimated. Handling buildings independently enables embarrassingly parallel job configurations on High Performance Computing (HPC) clusters. Such jobs scale very well and require minimal additional work to set up and run on a supercomputer. + +Performance Assessment Workflow +------------------------------- + +:numref:`figMainWorkflowComps` introduces the main parts and the generic workflow of the Pelicun framework and shows how its implementation connects to other modules in the SimCenter Application Framework. Each of the four highlighted models and their logical relationship are described in more detail in :numref:`figModelTypes`. + +.. _figMainWorkflowComps: + +.. figure:: figures/MainWorkflowComps.png + :align: center + :figclass: align-center + + The main components and the workflow of the Pelicun framework. + +.. _figModelTypes: + +.. figure:: figures/ModelTypes.png + :align: center + :figclass: align-center + + The four types of models and their logical relationships in the Pelicun framework. + +The calculation starts with two files: the Asset Information Model (AIM) and the EVENT file. Currently, both files are expected to follow a standard JSON file format defined by the SimCenter. Support of other file formats and data structures only require a custom parser method. The open source implementation of the framework can be extended by such a method and the following part of the calculation does not require any further adjustment. AIM is a generalized version of the widely used Building Information Model (BIM) idea and it holds structural, architectural, and performance-related information about an asset. The word asset is used to emphasize that the scope of Pelicun is not limited to building structures. The EVENT file describes the characteristic seismic events. It typically holds information about the frequency and intensity of the event, such as its occurrence rate or return period, and corresponding ground motion acceleration time histories or a collection of intensity measures. + +Two threads run in parallel and lead to the simulation of damage and losses: (a) response estimation, creating the response model, and simulation of EDPs; and (b) assembling the performance, damage, and loss models. In thread (a), the AIM and EVENT files are used to estimate the response of the asset to the seismic event and characterize it using EDPs. Peak interstory drift (PID), residual interstory drift (RID), and peak floor acceleration (PFA) are typically used as EDPs for building structures. Response simulation is out of the scope of Pelicun; it is either performed by the response estimation module in the Application Framework (Fig. 1) or it can be performed by any other application if Pelicun is used outside of the scope of SimCenter. The Pelicun framework can take advantage of response estimation methods that use idealized models for the seismic demand and the structural capacity, such as the capacity curve-based method in HAZUS or the regression-based closed-form approximation in the second edition of FEMA P-58 vol. 5 [12]. If the performance assessment follows path I or II from :numref:`figPerfAssWorkflows`, the estimated response is not needed, and the relevant IM values are used as EDPs. + +Response Model +-------------- + +The response model is based on the samples in the raw EDP file and provides a probabilistic description of the structural response. The samples can include an arbitrary number of EDP types (EDPt in Fig. 4) that describe the structural response at pre-defined locations and directions (EDPt,l,d). In buildings, locations typically correspond to floors or stories, and two directions are assigned to the primary and secondary horizontal axes. However, one might use more than two directions to collect several responses at each floor of an irregular building and locations can refer to other parts of structures, such as the piers of a bridge or segments of a pipeline. + +EDPs can be resampled either after fitting a probability distribution function to the raw data or by bootstrapping the raw EDPs. Besides the widely used multivariate lognormal distribution, its truncated version is also available. This allows the consideration, for example, that PID values above a pre-defined truncation limit are not reliable. Another option, using the raw EDPs as-is, is useful in regional simulations to preserve the order of samples and maintain the spatial dependencies introduced in random characteristics of the building inventory or the seismic hazard. + +Performance Model +----------------- + +Thread (b) in Fig. 3 starts with parsing the AIM file and constructing a performance model. If the definition in the file is incomplete, the auto-populate method tries to fill the gaps using information about normative component quantities and pre-defined rulesets. Rulesets can link structural information, such as the year of construction, to performance model details, such as the type of structural details and corresponding components. + +The performance model in Pelicun is based on that of the FEMA P-58 method. It disaggregates the asset into a hierarchical description of its structural and non-structural components and contents (Fig. 4): + +- Fragility Groups (FGs) are at the highest level of this hierarchy. Each FG is a collection of components that have similar fragility controlled by a specific type of EDP and their damage leads to similar consequences. + +- Each FG can be broken down into Performance Groups (PGs). A PG collects components whose damage is controlled by the same EDP. Not only the type of the EDP, but its location and direction also has to be identical. + +- In the third layer, PGs are broken down into the smallest units: Component Groups (CGs). A CG collects components that experience the same damage (i.e., there is perfect correlation between their random Damage States). Each CG has a Component Quantity assigned to it that defines the amount of components in that group. Both international standard and imperial units are supported as long as they are consistent with the type of component (e.g., m2, ft2, and in2 are all acceptable for the area of suspended ceilings, but ft is not.) Quantities can be random variables with either Normal or Lognormal distribution. + +In performance models built according to the FEMA P-58 method, buildings typically have FGs sensitive to either PID or PFA. Within each FG, components are grouped into PGs by stories and the drift-sensitive ones are also grouped by direction. The damage of acceleration-sensitive components is based on the maximum of PFAs in the two horizontal directions. The Applied Technology Council (ATC) provides a recommendation for the correlation between component damages within a PG. If the damages are correlated, all components in a PG are collected in a single CG. Otherwise, the performance model can identify an arbitrary number of CGs and their damages are evaluated independently. + +The Pelicun framework handles the probabilistic sampling for the entire performance model with a single high-dimensional random variable. This allows for custom dependencies in the model at any level of the hierarchy. For example, one can assign a 0.8 correlation coefficient between the fragility of all components in an FG that are on the same floor, but in different directions and hence, in different PGs. In another example, one can assign a 0.7 correlation coefficient between component quantities in the same direction along all or a subset of floors. These correlations can capture more realistic exposure and damage and consider the influence of extreme cases. Such cases are overlooked when independent variables are used because the deviations from the mean are cancelling each other. + +This performance model in Fig. 4 can also be applied to more holistic description of buildings. For example, to describe earthquake damage to buildings following HAZUS, three FGs can handle structural, acceleration-sensitive non-structural, and drift-sensitive non-structural components. Each FG has a single PG because HAZUS uses building-level EDPs—only one location and direction is used in this case. Since components describe the damage to the entire building, using one CG per PG with “1 ea” as the assigned, deterministic component quantity is appropriate. + +The performance model in Pelicun can facilitate filling the gap between the holistic and atomic approaches of performance assessment by using components at an intermediate resolution, such as story-based descriptions, for example. These models are promising because they require less detailed inputs than FEMA P-58, but they can provide more information than the building-level approaches in HAZUS. + +Damage Model +------------ + +Each Fragility Group in the performance model shall have a corresponding fragility model in the Damage & Loss Database. In the fragility model, Damage State Groups (DSGs) collect Damage States (DSs) that are triggered by similar magnitudes of the controlling EDP. In Pelicun, Lognormal damage state exceedance curves are converted into random EDP limits that trigger DSGs. When multiple DSGs are used, assuming perfect correlation between their EDP limits reproduces the conventional model that uses exceedance curves. The approach used in this framework, however, allows researchers to experiment with partially correlated or independent EDP limits. Experimental results suggest that these might be more realistic representations of component fragility. A DSG often has only a single DS. When multiple DSs are present, they can be triggered either simultaneously or they can be mutually exclusive following the corresponding definitions in FEMA P-58. + +Loss Model +---------- + +Each Damage State has a corresponding set of consequence descriptions in the Damage & Loss Database. These are used to define a consequence model that identifies a set of decision variables (DVs) and corresponding consequence functions that link the amount of damaged components to the value of the DV. The constant and quantity-dependent stepwise consequence functions from FEMA P-58 are available in Pelicun. + +Collapses and their consequences are also handled by the damage and the loss models. The collapse model describes collapse events using the concept of collapse modes introduced in FEMA P-58. Collapse is either triggered by EDP values exceeding a collapse limit or it can be randomly triggered based on a collapse probability prescribed in the AIM file. The latter approach allows for external collapse fragility models. Each collapse mode has a corresponding collapse consequence model that describes the corresponding injuries and losses. + +Similarly to the performance model, the randomness in damage and losses is handled with a few high-dimensional random variables. This allows researchers to experiment with various correlation structures between damages of components, and consequences of those damages. Among the consequences, the repair costs and times and the number of injuries of various severities are also linked; allowing, for example, to consider that repairs that cost more than expected will also take longer time to finish. + +Once the damage and loss models are assembled, the previously sampled EDPs are used to evaluate the Damage Measures (Fig. 3). These DMs identify the Damage State of each Component Group in the structure. This information is used by the loss simulation to generate the Decision Variables. The final step of the calculation in Pelicun is to aggregate results into a Damage and Loss (DL) file that provides a concise overview of the damage and losses. All intermediate data generated during the calculation (i.e., EDPs, DMs, DVs) are also saved in CSV files. diff --git a/doc/source/user_guide/resources_for_new_python_users.rst b/doc/source/user_guide/resources_for_new_python_users.rst new file mode 100644 index 000000000..0e96a1ec2 --- /dev/null +++ b/doc/source/user_guide/resources_for_new_python_users.rst @@ -0,0 +1,19 @@ +.. _new_python_users: + +Resources for new python users +------------------------------ + +The NHERI-SimCenter has hosted several programming bootcamps of which the reading material and recordings are available in the `NHERI-SimCenter Knowledge Hub `_. +For new Python users, we recommend the following resources: + +.. grid:: 1 2 2 2 + :gutter: 4 + :padding: 2 2 0 0 + :class-container: sd-text-center + + .. grid-item-card:: 2023 Programming Bootcamp > Python Quickstart Tutorial + :class-card: intro-card + :shadow: md + :link: https://nheri-simcenter.github.io/SimCenterBootcamp2023/source/lecture_videos_part1.html#chapter-6-modules-and-subprocess + + Training material on python basics, data types, loops, conditions, file IO, plotting, object oriented programming, classes, inheritance, modules. Presented by Peter Mackenzie-Helnwein. diff --git a/ignore_words.txt b/ignore_words.txt new file mode 100644 index 000000000..d2d0a1f68 --- /dev/null +++ b/ignore_words.txt @@ -0,0 +1,2 @@ +smoot +ACI diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index ead6d5760..000000000 --- a/mypy.ini +++ /dev/null @@ -1,3 +0,0 @@ -[mypy] -ignore_missing_imports = True -exclude = "^flycheck\." diff --git a/pelicun/__init__.py b/pelicun/__init__.py index 5497718a5..a2cebd886 100644 --- a/pelicun/__init__.py +++ b/pelicun/__init__.py @@ -1,52 +1,50 @@ -""" --*- coding: utf-8 -*- +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California -Copyright (c) 2018 Leland Stanford Junior University -Copyright (c) 2018 The Regents of the University of California +# This file is part of pelicun. -This file is part of pelicun. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. -1. Redistributions of source code must retain the above copyright notice, -this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. -3. Neither the name of the copyright holder nor the names of its contributors -may be used to endorse or promote products derived from this software without -specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . -You should have received a copy of the BSD 3-Clause License along with -pelicun. If not, see . +# Contributors: +# Adam Zsarnóczay -Contributors: -Adam Zsarnóczay -""" +"""Pelicun library.""" -name = "pelicun" +name = 'pelicun' -__version__ = '3.3.2' +__version__ = '3.4.0' __copyright__ = ( - "Copyright (c) 2018 Leland Stanford " - "Junior University and The Regents " - "of the University of California" + 'Copyright (c) 2018 Leland Stanford ' + 'Junior University and The Regents ' + 'of the University of California' ) -__license__ = "BSD 3-Clause License" +__license__ = 'BSD 3-Clause License' diff --git a/pelicun/assessment.py b/pelicun/assessment.py index 673baad35..8525c2f1d 100644 --- a/pelicun/assessment.py +++ b/pelicun/assessment.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -38,72 +37,97 @@ # Adam Zsarnóczay # John Vouvakis Manousakis -""" -This module has classes and methods that control the performance assessment. -.. rubric:: Contents - -.. autosummary:: - - Assessment - -""" +"""Classes and methods that control the performance assessment.""" from __future__ import annotations -from typing import Any + import json +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import numpy as np import pandas as pd -from pelicun import base -from pelicun import file_io -from pelicun import model + +from pelicun import base, file_io, model, uq from pelicun.__init__ import __version__ as pelicun_version # type: ignore +from pelicun.base import EDP_to_demand_type, get + +if TYPE_CHECKING: + from pelicun.base import Logger +default_dbs = { + 'fragility': { + 'FEMA P-58': 'damage_DB_FEMA_P58_2nd.csv', + 'Hazus Earthquake - Buildings': 'damage_DB_Hazus_EQ_bldg.csv', + 'Hazus Earthquake - Stories': 'damage_DB_Hazus_EQ_story.csv', + 'Hazus Earthquake - Transportation': 'damage_DB_Hazus_EQ_trnsp.csv', + 'Hazus Earthquake - Water': 'damage_DB_Hazus_EQ_water.csv', + 'Hazus Hurricane': 'damage_DB_SimCenter_Hazus_HU_bldg.csv', + }, + 'repair': { + 'FEMA P-58': 'loss_repair_DB_FEMA_P58_2nd.csv', + 'Hazus Earthquake - Buildings': 'loss_repair_DB_Hazus_EQ_bldg.csv', + 'Hazus Earthquake - Stories': 'loss_repair_DB_Hazus_EQ_story.csv', + 'Hazus Earthquake - Transportation': 'loss_repair_DB_Hazus_EQ_trnsp.csv', + 'Hazus Hurricane': 'loss_repair_DB_SimCenter_Hazus_HU_bldg.csv', + }, +} -class Assessment: +default_damage_processes = { + 'FEMA P-58': { + '1_excessive.coll.DEM': {'DS1': 'collapse_DS1'}, + '2_collapse': {'DS1': 'ALL_NA'}, + '3_excessiveRID': {'DS1': 'irreparable_DS1'}, + }, + # TODO(AZ): expand with ground failure logic + 'Hazus Earthquake': { + '1_STR': {'DS5': 'collapse_DS1'}, + '2_LF': {'DS5': 'collapse_DS1'}, + '3_excessive.coll.DEM': {'DS1': 'collapse_DS1'}, + '4_collapse': {'DS1': 'ALL_NA'}, + '5_excessiveRID': {'DS1': 'irreparable_DS1'}, + }, + 'Hazus Hurricane': {}, +} + + +class AssessmentBase: """ + Base class for Assessment objects. + Assessment objects manage the models, data, and calculations in pelicun. - Parameters - ---------- - demand: DemandModel - ... - asset: AssetModel - ... - damage: DamageModel - ... - repair: RepairModel - ... - stories: int - Number of stories. - options: Options - Options object. """ - __slots__ = [ - 'stories', - 'options', - 'unit_conversion_factors', - 'log', - 'demand', + __slots__: list[str] = [ 'asset', 'damage', + 'demand', + 'log', 'loss', + 'options', + 'stories', + 'unit_conversion_factors', ] - def __init__(self, config_options: dict[str, Any] | None = None): + def __init__(self, config_options: dict[str, Any] | None = None) -> None: """ - Initializes an Assessment object. + Initialize an Assessment object. Parameters ---------- - config_options (Optional[dict]): + config_options: User-specified configuration dictionary. + """ - self.stories = None + self.stories: int | None = None self.options = base.Options(config_options, self) - self.unit_conversion_factors = base.parse_units(self.options.units_file) + self.unit_conversion_factors: dict = base.parse_units( + self.options.units_file + ) - self.log = self.options.log + self.log: Logger = self.options.log self.log.msg( f'pelicun {pelicun_version} | \n', prepend_timestamp=False, @@ -119,9 +143,9 @@ def __init__(self, config_options: dict[str, Any] | None = None): self.loss: model.LossModel = model.LossModel(self) @property - def bldg_repair(self): + def bldg_repair(self) -> model.LossModel: """ - + Exists for . Returns ------- @@ -129,7 +153,7 @@ def bldg_repair(self): The loss model. """ - self.log.warn( + self.log.warning( '`.bldg_repair` is deprecated and will be dropped in ' 'future versions of pelicun. ' 'Please use `.loss` instead.' @@ -138,9 +162,9 @@ def bldg_repair(self): return self.loss @property - def repair(self): + def repair(self) -> model.LossModel: """ - + Exists for . Returns ------- @@ -148,7 +172,7 @@ def repair(self): The damage state-driven component loss model. """ - self.log.warn( + self.log.warning( '`.repair` is deprecated and will be dropped in ' 'future versions of pelicun. ' 'Please use `.loss` instead.' @@ -157,6 +181,8 @@ def repair(self): def get_default_data(self, data_name: str) -> pd.DataFrame: """ + Load a default data file. + Loads a default data file by name and returns it. This method is specifically designed to access predefined CSV files from a structured directory path related to the SimCenter fragility @@ -164,7 +190,7 @@ def get_default_data(self, data_name: str) -> pd.DataFrame: Parameters ---------- - data_name : str + data_name: str The name of the CSV file to be loaded, without the '.csv' extension. This name is used to construct the full path to the file. @@ -174,19 +200,19 @@ def get_default_data(self, data_name: str) -> pd.DataFrame: pd.DataFrame The DataFrame containing the data loaded from the specified CSV file. - """ + """ # if 'fragility_DB' in data_name: data_name = data_name.replace('fragility_DB', 'damage_DB') - self.log.warn( + self.log.warning( '`fragility_DB` is deprecated and will be dropped in ' 'future versions of pelicun. ' 'Please use `damage_DB` instead.' ) if 'bldg_repair_DB' in data_name: data_name = data_name.replace('bldg_repair_DB', 'loss_repair_DB') - self.log.warn( + self.log.warning( '`bldg_repair_DB` is deprecated and will be dropped in ' 'future versions of pelicun. ' 'Please use `loss_repair_DB` instead.' @@ -194,10 +220,13 @@ def get_default_data(self, data_name: str) -> pd.DataFrame: data_path = f'{base.pelicun_path}/resources/SimCenterDBDL/{data_name}.csv' - return file_io.load_data( + data = file_io.load_data( data_path, None, orientation=1, reindex=False, log=self.log ) + assert isinstance(data, pd.DataFrame) + return data + def get_default_metadata(self, data_name: str) -> dict: """ Load a default metadata file and pass it to the user. @@ -213,25 +242,26 @@ def get_default_metadata(self, data_name: str) -> dict: Default metadata """ - # if 'fragility_DB' in data_name: data_name = data_name.replace('fragility_DB', 'damage_DB') - self.log.warn( + self.log.warning( '`fragility_DB` is deprecated and will be dropped in ' 'future versions of pelicun. Please use `damage_DB` instead.' ) data_path = f'{base.pelicun_path}/resources/SimCenterDBDL/{data_name}.json' - with open(data_path, 'r', encoding='utf-8') as f: + with Path(data_path).open(encoding='utf-8') as f: data = json.load(f) - return data + return data # noqa: RET504 def calc_unit_scale_factor(self, unit: str) -> float: """ + Determine unit scale factor. + Determines the scale factor from input unit to the - corresponding base unit + corresponding base unit. Parameters ---------- @@ -249,14 +279,14 @@ def calc_unit_scale_factor(self, unit: str) -> float: ------ KeyError When an invalid unit is specified - """ + """ unit_lst = unit.strip().split(' ') # check if there is a quantity specified; if yes, parse it if len(unit_lst) > 1: - unit_count, unit_name = unit_lst - unit_count = float(unit_count) + unit_count_str, unit_name = unit_lst + unit_count = float(unit_count_str) else: unit_count = 1 @@ -266,14 +296,15 @@ def calc_unit_scale_factor(self, unit: str) -> float: scale_factor = unit_count * self.unit_conversion_factors[unit_name] except KeyError as exc: - raise KeyError( - f"Specified unit not recognized: {unit_count} {unit_name}" - ) from exc + msg = f'Specified unit not recognized: {unit_count} {unit_name}' + raise KeyError(msg) from exc return scale_factor def scale_factor(self, unit: str | None) -> float: """ + Get scale factor of given unit. + Returns the scale factor of a given unit. If the unit is unknown it raises an error. If the unit is None it returns 1.00. @@ -294,14 +325,1627 @@ def scale_factor(self, unit: str | None) -> float: If the unit is unknown. """ - if unit is not None: if unit in self.unit_conversion_factors: scale_factor = self.unit_conversion_factors[unit] else: - raise ValueError(f"Unknown unit: {unit}") + msg = f'Unknown unit: {unit}' + raise ValueError(msg) else: scale_factor = 1.0 return scale_factor + + +class Assessment(AssessmentBase): + """ + Assessment class. + + Has methods implementing a Scenario-Based assessment. + + """ + + __slots__: list[str] = [] + + def calculate_damage( + self, + num_stories: int, + demand_config: dict, + demand_data_source: str | dict, + cmp_data_source: str | dict[str, pd.DataFrame], + damage_data_paths: list[str | pd.DataFrame], + dmg_process: dict | None = None, + scaling_specification: dict | None = None, + residual_drift_configuration: dict | None = None, + collapse_fragility_configuration: dict | None = None, + block_batch_size: int = 1000, + ) -> None: + """ + Calculate damage. + + Parameters + ---------- + num_stories: int + Number of stories of the asset. Applicable to buildings. + demand_config: dict + A dictionary containing configuration options for the + sample generation. Key options include: + * 'SampleSize': The number of samples to generate. + * 'PreserveRawOrder': Boolean indicating whether to + preserve the order of the raw data. Defaults to False. + * 'DemandCloning': Specifies if and how demand cloning + should be applied. Can be a boolean or a detailed + configuration. + demand_data_source: string or dict + If string, the demand_data_source is a file prefix + ( in the following description) that identifies + the following files: _marginals.csv, + _empirical.csv, _correlation.csv. If dict, + the demand data source is a dictionary with the following + optional keys: 'marginals', 'empirical', and + 'correlation'. The value under each key shall be a + DataFrame. + cmp_data_source: str or dict + The source from where to load the component model data. If + it's a string, it should be the prefix for three files: + one for marginal distributions (`_marginals.csv`), + one for empirical data (`_empirical.csv`), and one + for correlation data (`_correlation.csv`). If it's + a dictionary, it should have keys 'marginals', + 'empirical', and 'correlation', with each key associated + with a DataFrame containing the corresponding data. + damage_data_paths: list of (string | DataFrame) + List of paths to data or files with damage model + information. Default XY datasets can be accessed as + PelicunDefault/XY. Order matters. Parameters defined in + prior elements in the list take precedence over the same + parameters in subsequent data paths. I.e., place the + Default datasets in the back. + dmg_process: dict, optional + Allows simulating damage processes, where damage to some + component can alter the damage state of other components. + scaling_specification: dict, optional + A dictionary defining the shift in median. + Example: {'CMP-1-1': '*1.2', 'CMP-1-2': '/1.4'} + The keys are individual components that should be present + in the `capacity_sample`. The values should be strings + containing an operation followed by the value formatted as + a float. The operation can be '+' for addition, '-' for + subtraction, '*' for multiplication, and '/' for division. + residual_drift_configuration: dict + Dictionary containing the following keys-values: + - params: dict + A dictionary containing parameters required for the + estimation method, such as 'yield_drift', which is the + drift at which yielding is expected to occur. + - method: str, optional + The method used to estimate the RID values. Currently, + only 'FEMA P58' is implemented. Defaults to 'FEMA P58'. + collapse_fragility_configuration: dict + Dictionary containing the following keys-values: + - label: str + Label to use to extend the MultiIndex of the demand + sample. + - value: float + Values to add to the rows of the additional column. + - unit: str + Unit that corresponds to the additional column. + - location: str, optional + Optional location, defaults to `0`. + - direction: str, optional + Optional direction, defaults to `1`. + block_batch_size: int + Maximum number of components in each batch. + + """ + # TODO(JVM): when we build the API docs, ensure the above is + # properly rendered. + + self.demand.load_model(demand_data_source) + self.demand.generate_sample(demand_config) + + if residual_drift_configuration: + self.demand.estimate_RID_and_adjust_sample( + residual_drift_configuration['parameters'], + residual_drift_configuration['method'], + ) + + if collapse_fragility_configuration: + self.demand.expand_sample( + collapse_fragility_configuration['label'], + collapse_fragility_configuration['value'], + collapse_fragility_configuration['unit'], + ) + + self.stories = num_stories + self.asset.load_cmp_model(cmp_data_source) + self.asset.generate_cmp_sample() + + self.damage.load_model_parameters( + damage_data_paths, set(self.asset.list_unique_component_ids()) + ) + self.damage.calculate(dmg_process, block_batch_size, scaling_specification) + + def calculate_loss( + self, + decision_variables: tuple[str, ...], + loss_model_data_paths: list[str | pd.DataFrame], + loss_map_path: str | pd.DataFrame | None = None, + loss_map_policy: str | None = None, + ) -> None: + """ + Calculate loss. + + Parameters + ---------- + decision_variables: tuple + Defines the decision variables to be included in the loss + calculations. Defaults to those supported, but fewer can be + used if desired. When fewer are used, the loss parameters for + those not used will not be required. + loss_model_data_paths: list of (string | DataFrame) + List of paths to data or files with loss model + information. Default XY datasets can be accessed as + PelicunDefault/XY. Order matters. Parameters defined in + prior elements in the list take precedence over the same + parameters in subsequent data paths. I.e., place the + Default datasets in the back. + loss_map_path: str or pd.DataFrame or None + Path to a csv file or DataFrame object that maps + components IDs to their loss parameter definitions. + loss_map_policy: str or None + If None, does not modify the loss map. + If set to `fill`, each component ID that is present in + the asset model but not in the loss map is mapped to + itself, but `excessiveRID` is excluded. + If set to `fill_all`, each component ID that is present in + the asset model but not in the loss map is mapped to + itself without exceptions. + + """ + self.loss.decision_variables = decision_variables + self.loss.add_loss_map(loss_map_path, loss_map_policy) + self.loss.load_model_parameters(loss_model_data_paths) + self.loss.calculate() + + def aggregate_loss( + self, + replacement_configuration: ( + tuple[uq.RandomVariableRegistry, dict[str, float]] | None + ) = None, + loss_combination: dict | None = None, + ) -> tuple[pd.DataFrame, pd.DataFrame]: + """ + Aggregate losses. + + Parameters + ---------- + replacement_configuration: Tuple, optional + Tuple containing a RandomVariableRegistry and a + dictionary. The RandomVariableRegistry is defining + building replacement consequence RVs for the active + decision variables. The dictionary defines exceedance + thresholds. If the aggregated value for a decision + variable (conditioned on no replacement) exceeds the + threshold, then replacement is triggered. This can happen + for multiple decision variables at the same + realization. The consequence keyword `replacement` is + reserved to represent exclusive triggering of the + replacement consequences, and other consequences are + ignored for those realizations where replacement is + triggered. When assigned to None, then `replacement` is + still treated as an exclusive consequence (other + consequences are set to zero when replacement is nonzero) + but it is not being additionally triggered by the + exceedance of any thresholds. The aggregated loss sample + contains an additional column with information on whether + replacement was already present or triggered by a + threshold exceedance for each realization. + loss_combination: dict, optional + Dictionary defining how losses for specific components + should be aggregated for a given decision variable. It has + the following structure: {`dv`: {(`c1`, `c2`): `arr`, + ...}, ...}, where `dv` is some decision variable, (`c1`, + `c2`) is a tuple defining a component pair, `arr` is a NxN + numpy array defining a combination table, and `...` means + that more key-value pairs with the same schema can exist + in the dictionaries. The loss sample is expected to + contain columns that include both `c1` and `c2` listed as + the component. The combination is applied to all pairs of + columns where the components are `c1` and `c2`, and all of + the rest of the multiindex levels match (`loc`, `dir`, + `uid`). This means, for example, that when combining wind + and flood losses, the asset model should contain both a + wind and a flood component defined at the same + location-direction. `arr` can also be an M-dimensional + numpy array where each dimension has length N (NxNx...xN). + This structure allows for the loss combination of M + components. In this case the (`c1`, `c2`) tuple should + contain M elements instead of two. + + Notes + ----- + Regardless of the value of the arguments, this method does not + alter the state of the loss model, i.e., it does not modify + the values of the `.sample` attributes. + + Returns + ------- + tuple + Dataframe with the aggregated loss of each realization, + and another boolean dataframe with information on which DV + thresholds were exceeded in each realization, triggering + replacement. If no thresholds are specified it only + contains False values. + + """ + output = self.loss.aggregate_losses( + replacement_configuration, loss_combination, future=True + ) + assert isinstance(output, tuple) + return output + + +class DLCalculationAssessment(AssessmentBase): + """Base class for the assessment objects used in `DL_calculation.py`.""" + + __slots__: list[str] = [] + + def calculate_demand( # noqa: C901 + self, + demand_path: Path, + collapse_limits: dict[str, float] | None, + length_unit: str | None, + demand_calibration: dict | None, + sample_size: int, + demand_cloning: dict | None, + residual_drift_inference: dict | None, + *, + coupled_demands: bool, + ) -> None: + """ + Calculate demands. + + Parameters + ---------- + demand_path: str + Path to the demand data file. + collapse_limits: dict[str, float] or None + Optional dictionary with demand types and their respective + collapse limits. + length_unit : str, optional + Unit of length to be used to add units to the demand data + if needed. + demand_calibration: dict or None + Calibration data for the demand model. + sample_size: int + Number of realizations. + coupled_demands: bool + Whether to preserve the raw order of the demands. + demand_cloning: dict or None + Demand cloning configuration. + residual_drift_inference: dict or None + Information for residual drift inference. + + Raises + ------ + ValueError + When an unknown residual drift method is specified. + + """ + idx = pd.IndexSlice + raw_demands = pd.read_csv(demand_path, index_col=0) + + # remove excessive demands that are considered collapses, if needed + if collapse_limits: + raw_demands_m = base.convert_to_MultiIndex(raw_demands, axis=1) + assert isinstance(raw_demands_m, pd.DataFrame) + raw_demands = raw_demands_m + + if 'Units' in raw_demands.index: + raw_units = raw_demands.loc['Units', :] + raw_demands = raw_demands.drop('Units', axis=0).astype(float) + + else: + raw_units = None + + dem_to_drop = np.full(raw_demands.shape[0], fill_value=False) + + for dem_type, limit in collapse_limits.items(): + assert isinstance(dem_type, str) + assert isinstance(limit, (str, float)) + nlevels_with_event_id = 4 + if raw_demands.columns.nlevels == nlevels_with_event_id: + dem_to_drop += raw_demands.loc[ + :, # type: ignore + idx[:, dem_type, :, :], + ].max(axis=1) > float(limit) + + else: + dem_to_drop += raw_demands.loc[ + :, # type: ignore + idx[dem_type, :, :], + ].max(axis=1) > float(limit) + + raw_demands = raw_demands.loc[~dem_to_drop, :] + + if isinstance(raw_units, pd.Series): + raw_demands = pd.concat( + [raw_demands, raw_units.to_frame().T], axis=0 + ) + + self.log.msg( + f'{np.sum(dem_to_drop)} realizations removed from the demand ' + f'input because they exceed the collapse limit. The remaining ' + f'sample size: {raw_demands.shape[0]}' + ) + + # add units to the demand data if needed + if 'Units' not in raw_demands.index: + if length_unit is None: + msg = 'A length unit is required to infer demand units.' + raise ValueError(msg) + demands = _add_units(raw_demands, length_unit) + + else: + demands = raw_demands + + # load the available demand sample + self.demand.load_sample(demands) + + # get the calibration information + if demand_calibration: + # then use it to calibrate the demand model + self.demand.calibrate_model(demand_calibration) + + else: + # if no calibration is requested, + # set all demands to use empirical distribution + self.demand.calibrate_model({'ALL': {'DistributionFamily': 'empirical'}}) + + # and generate a new demand sample + self.demand.generate_sample( + { + 'SampleSize': sample_size, + 'PreserveRawOrder': coupled_demands, + 'DemandCloning': demand_cloning, + } + ) + + # get the generated demand sample + demand_sample_tuple = self.demand.save_sample(save_units=True) + assert demand_sample_tuple is not None + demand_sample, demand_units = demand_sample_tuple + assert isinstance(demand_sample, pd.DataFrame) + assert isinstance(demand_units, pd.Series) + + demand_sample = pd.concat([demand_sample, demand_units.to_frame().T]) + + # get residual drift estimates, if needed + if residual_drift_inference: + # `method` is guaranteed to exist because it is confirmed when + # parsing the configuration file. + rid_inference_method = residual_drift_inference.pop('method') + + if rid_inference_method == 'FEMA P-58': + rid_list: list[pd.DataFrame] = [] + pid = demand_sample['PID'].copy() + pid = pid.drop('Units') + pid = pid.astype(float) + + for direction, delta_yield in residual_drift_inference.items(): + pids = pid.loc[:, idx[:, direction]] # type: ignore + assert isinstance(pids, pd.DataFrame) + rid = self.demand.estimate_RID( + pids, + {'yield_drift': float(delta_yield)}, + ) + + rid_list.append(rid) + + rid = pd.concat(rid_list, axis=1) + rid_units = pd.Series( + ['unitless'] * rid.shape[1], + index=rid.columns, + name='Units', + ) + rid_sample = pd.concat([rid, rid_units.to_frame().T]) + demand_sample = pd.concat([demand_sample, rid_sample], axis=1) + + else: + msg = ( + f'Unknown residual drift inference method: ' + f'`{rid_inference_method}`.' + ) + raise ValueError(msg) + + # add a constant one demand + demand_sample['ONE', '0', '1'] = np.ones(demand_sample.shape[0]) + demand_sample.loc['Units', ('ONE', '0', '1')] = 'unitless' + + self.demand.load_sample(base.convert_to_SimpleIndex(demand_sample, axis=1)) + + def calculate_asset( + self, + num_stories: int, + component_assignment_file: str | None, + collapse_fragility_demand_type: str | None, + component_sample_file: str | None, + *, + add_irreparable_damage_columns: bool, + ) -> None: + """ + Generate the asset model sample. + + Parameters + ---------- + num_stories: int + Number of stories. + component_assignment_file: str or None + Path to a component assignment file. + collapse_fragility_demand_type: str or None + Optional demand type for the collapse fragility. + add_irreparable_damage_columns: bool + Whether to add columns for irreparable damage. + component_sample_file: str or None + Optional path to an existing component sample file. + + Raises + ------ + ValueError + With invalid combinations of arguments. + + """ + # retrieve the demand sample + demand_sample = self.demand.save_sample() + assert isinstance(demand_sample, pd.DataFrame) + + # set the number of stories + if num_stories: + self.stories = num_stories + + # We either accept a `component_assignment_file` or a + # `component_sample_file`, not both. + if ( + component_assignment_file is not None + and component_sample_file is not None + ): + msg = ( + 'Both `component_assignment_file` and ' + '`component_sample_file` are provided. ' + 'Please provide only one.' + ) + raise ValueError(msg) + + # load a component model and generate a sample + if component_assignment_file is not None: + cmp_marginals = pd.read_csv( + component_assignment_file, + index_col=0, + encoding_errors='replace', + ) + + dem_types = demand_sample.columns.unique(level=0) + + # add component(s) to support collapse calculation + if collapse_fragility_demand_type is not None: + if not collapse_fragility_demand_type.startswith('SA'): + # we need story-specific collapse assessment + # (otherwise we have a global demand and evaluate + # collapse directly, so this code should be skipped) + + if collapse_fragility_demand_type in dem_types: + # excessive coll_DEM is added on every floor + # to detect large RIDs + cmp_marginals.loc['excessive.coll.DEM', 'Units'] = 'ea' + + locs = demand_sample[ + collapse_fragility_demand_type # type: ignore + ].columns.unique(level=0) + cmp_marginals.loc['excessive.coll.DEM', 'Location'] = ( + ','.join(locs) + ) + + dirs = demand_sample[ + collapse_fragility_demand_type # type: ignore + ].columns.unique(level=1) + cmp_marginals.loc['excessive.coll.DEM', 'Direction'] = ( + ','.join(dirs) + ) + + cmp_marginals.loc['excessive.coll.DEM', 'Theta_0'] = 1.0 + + else: + self.log.msg( + f'WARNING: No {collapse_fragility_demand_type} ' + f'among available demands. Collapse cannot ' + f'be evaluated.' + ) + + # always add a component to support basic collapse calculation + cmp_marginals.loc['collapse', 'Units'] = 'ea' + cmp_marginals.loc['collapse', 'Location'] = 0 + cmp_marginals.loc['collapse', 'Direction'] = 1 + cmp_marginals.loc['collapse', 'Theta_0'] = 1.0 + + # add components to support irreparable damage calculation + if add_irreparable_damage_columns: + if 'RID' in dem_types: + # excessive RID is added on every floor to detect large RIDs + cmp_marginals.loc['excessiveRID', 'Units'] = 'ea' + + locs = demand_sample['RID'].columns.unique(level=0) + cmp_marginals.loc['excessiveRID', 'Location'] = ','.join(locs) + + dirs = demand_sample['RID'].columns.unique(level=1) + cmp_marginals.loc['excessiveRID', 'Direction'] = ','.join(dirs) + + cmp_marginals.loc['excessiveRID', 'Theta_0'] = 1.0 + + # irreparable is a global component to recognize is any of the + # excessive RIDs were triggered + cmp_marginals.loc['irreparable', 'Units'] = 'ea' + cmp_marginals.loc['irreparable', 'Location'] = 0 + cmp_marginals.loc['irreparable', 'Direction'] = 1 + cmp_marginals.loc['irreparable', 'Theta_0'] = 1.0 + + else: + self.log.msg( + 'WARNING: No residual interstory drift ratio among ' + 'available demands. Irreparable damage cannot be ' + 'evaluated.' + ) + + # load component model + self.asset.load_cmp_model({'marginals': cmp_marginals}) + + # generate component quantity sample + self.asset.generate_cmp_sample() + + # if requested, load the quantity sample from a file + if component_sample_file is not None: + self.asset.load_cmp_sample(component_sample_file) + + def calculate_damage( # noqa: C901 + self, + length_unit: str | None, + component_database: str, + component_database_path: str | None = None, + collapse_fragility: dict | None = None, + irreparable_damage: dict | None = None, + damage_process_approach: str | None = None, + damage_process_file_path: str | None = None, + custom_model_dir: str | None = None, + scaling_specification: dict | None = None, + *, + is_for_water_network_assessment: bool = False, + ) -> None: + """ + Calculate damage. + + Parameters + ---------- + length_unit : str, optional + Unit of length to be used to add units to the demand data + if needed. + component_database: str + Name of the component database. + component_database_path: str or None + Optional path to a component database file. + collapse_fragility: dict or None + Collapse fragility information. + irreparable_damage: dict or None + Information for irreparable damage. + damage_process_approach: str or None + Approach for the damage process. + damage_process_file_path: str or None + Optional path to a damage process file. + custom_model_dir: str or None + Optional directory for custom models. + scaling_specification: dict, optional + A dictionary defining the shift in median. + Example: {'CMP-1-1': '*1.2', 'CMP-1-2': '/1.4'} + The keys are individual components that should be present + in the `capacity_sample`. The values should be strings + containing an operation followed by the value formatted as + a float. The operation can be '+' for addition, '-' for + subtraction, '*' for multiplication, and '/' for division. + is_for_water_network_assessment: bool + Whether the assessment is for a water network. + + Raises + ------ + ValueError + With invalid combinations of arguments. + + """ + # load the fragility information + if component_database in default_dbs['fragility']: + component_db = [ + 'PelicunDefault/' + default_dbs['fragility'][component_database], + ] + else: + component_db = [] + + if component_database_path is not None: + if custom_model_dir is None: + msg = ( + '`custom_model_dir` needs to be specified ' + 'when `component_database_path` is not None.' + ) + raise ValueError(msg) + + if 'CustomDLDataFolder' in component_database_path: + component_database_path = component_database_path.replace( + 'CustomDLDataFolder', custom_model_dir + ) + + component_db += [component_database_path] + + component_db = component_db[::-1] + + # prepare additional fragility data + + # get the database header from the default P58 db + p58_data = self.get_default_data('damage_DB_FEMA_P58_2nd') + + adf = pd.DataFrame(columns=p58_data.columns) + + if collapse_fragility: + assert self.asset.cmp_marginal_params is not None + + if ( + 'excessive.coll.DEM' + in self.asset.cmp_marginal_params.index.get_level_values('cmp') + ): + # if there is story-specific evaluation + coll_cmp_name = 'excessive.coll.DEM' + else: + # otherwise, for global collapse evaluation + coll_cmp_name = 'collapse' + + adf.loc[coll_cmp_name, ('Demand', 'Directional')] = 1 + adf.loc[coll_cmp_name, ('Demand', 'Offset')] = 0 + + coll_dem = collapse_fragility['DemandType'] + + if '_' in coll_dem: + coll_dem, coll_dem_spec = coll_dem.split('_') + else: + coll_dem_spec = None + + coll_dem_name = None + for demand_name, demand_short in EDP_to_demand_type.items(): + if demand_short == coll_dem: + coll_dem_name = demand_name + break + + if coll_dem_name is None: + msg = ( + 'A valid demand type acronym was not provided in' + 'the configuration file. Please ensure the' + "'DemandType' field in the collapse fragility" + 'section contains one of the recognized acronyms' + "(e.g., 'SA', 'PFA', 'PGA'). Refer to the" + "configuration file's 'collapse_fragility'" + 'section.' + ) + raise ValueError(msg) + + if coll_dem_spec is None: + adf.loc[coll_cmp_name, ('Demand', 'Type')] = coll_dem_name + + else: + adf.loc[coll_cmp_name, ('Demand', 'Type')] = ( + f'{coll_dem_name}|{coll_dem_spec}' + ) + + if length_unit is None: + msg = 'A length unit is required.' + raise ValueError(msg) + coll_dem_unit = _add_units( + pd.DataFrame( + columns=[ + f'{coll_dem}-1-1', + ] + ), + length_unit, + ).iloc[0, 0] + + adf.loc[coll_cmp_name, ('Demand', 'Unit')] = coll_dem_unit + adf.loc[coll_cmp_name, ('LS1', 'Family')] = collapse_fragility[ + 'CapacityDistribution' + ] + adf.loc[coll_cmp_name, ('LS1', 'Theta_0')] = collapse_fragility[ + 'CapacityMedian' + ] + adf.loc[coll_cmp_name, ('LS1', 'Theta_1')] = collapse_fragility[ + 'Theta_1' + ] + adf.loc[coll_cmp_name, 'Incomplete'] = 0 + + if coll_cmp_name != 'collapse': + # for story-specific evaluation, we need to add a placeholder + # fragility that will never trigger, but helps us aggregate + # results in the end + adf.loc['collapse', ('Demand', 'Directional')] = 1 + adf.loc['collapse', ('Demand', 'Offset')] = 0 + adf.loc['collapse', ('Demand', 'Type')] = 'One' + adf.loc['collapse', ('Demand', 'Unit')] = 'unitless' + adf.loc['collapse', ('LS1', 'Theta_0')] = 1e10 + adf.loc['collapse', 'Incomplete'] = 0 + + elif not is_for_water_network_assessment: + # add a placeholder collapse fragility that will never trigger + # collapse, but allow damage processes to work with collapse + + adf.loc['collapse', ('Demand', 'Directional')] = 1 + adf.loc['collapse', ('Demand', 'Offset')] = 0 + adf.loc['collapse', ('Demand', 'Type')] = 'One' + adf.loc['collapse', ('Demand', 'Unit')] = 'unitless' + adf.loc['collapse', ('LS1', 'Theta_0')] = 1e10 + adf.loc['collapse', 'Incomplete'] = 0 + + if irreparable_damage: + # add excessive RID fragility according to settings provided in the + # input file + adf.loc['excessiveRID', ('Demand', 'Directional')] = 1 + adf.loc['excessiveRID', ('Demand', 'Offset')] = 0 + adf.loc['excessiveRID', ('Demand', 'Type')] = ( + 'Residual Interstory Drift Ratio' + ) + + adf.loc['excessiveRID', ('Demand', 'Unit')] = 'unitless' + adf.loc['excessiveRID', ('LS1', 'Theta_0')] = irreparable_damage[ + 'DriftCapacityMedian' + ] + adf.loc['excessiveRID', ('LS1', 'Family')] = 'lognormal' + adf.loc['excessiveRID', ('LS1', 'Theta_1')] = irreparable_damage[ + 'DriftCapacityLogStd' + ] + + adf.loc['excessiveRID', 'Incomplete'] = 0 + + # add a placeholder irreparable fragility that will never trigger + # damage, but allow damage processes to aggregate excessiveRID here + adf.loc['irreparable', ('Demand', 'Directional')] = 1 + adf.loc['irreparable', ('Demand', 'Offset')] = 0 + adf.loc['irreparable', ('Demand', 'Type')] = 'One' + adf.loc['irreparable', ('Demand', 'Unit')] = 'unitless' + adf.loc['irreparable', ('LS1', 'Theta_0')] = 1e10 + adf.loc['irreparable', 'Incomplete'] = 0 + + # TODO(AZ): we can improve this by creating a water + # network-specific assessment class + if is_for_water_network_assessment: + # add a placeholder aggregate fragility that will never trigger + # damage, but allow damage processes to aggregate the + # various pipeline damages + adf.loc['aggregate', ('Demand', 'Directional')] = 1 + adf.loc['aggregate', ('Demand', 'Offset')] = 0 + adf.loc['aggregate', ('Demand', 'Type')] = 'Peak Ground Velocity' + adf.loc['aggregate', ('Demand', 'Unit')] = 'mps' + adf.loc['aggregate', ('LS1', 'Theta_0')] = 1e10 + adf.loc['aggregate', ('LS2', 'Theta_0')] = 1e10 + adf.loc['aggregate', 'Incomplete'] = 0 + + self.damage.load_model_parameters( + [*component_db, adf], + set(self.asset.list_unique_component_ids()), + ) + + # load the damage process if needed + dmg_process = None + if damage_process_approach is not None: # noqa: PLR1702 + if damage_process_approach in default_damage_processes: + dmg_process = default_damage_processes[damage_process_approach] + + # For Hazus Earthquake, we need to specify the component ids + if damage_process_approach == 'Hazus Earthquake': + cmp_sample = self.asset.save_cmp_sample() + assert isinstance(cmp_sample, pd.DataFrame) + + cmp_list = cmp_sample.columns.unique(level=0) + + cmp_map = {'STR': '', 'LF': '', 'NSA': ''} + + for cmp in cmp_list: + for cmp_type in cmp_map: + if cmp_type + '.' in cmp: + cmp_map[cmp_type] = cmp + + new_dmg_process = dmg_process.copy() + for source_cmp, action in dmg_process.items(): + # first, look at the source component id + new_source = None + for cmp_type, cmp_id in cmp_map.items(): + if (cmp_type in source_cmp) and (cmp_id != ''): # noqa: PLC1901 + new_source = source_cmp.replace(cmp_type, cmp_id) + break + + if new_source is not None: + new_dmg_process[new_source] = action + del new_dmg_process[source_cmp] + else: + new_source = source_cmp + + # then, look at the target component ids + for ds_i, target_vals in action.items(): + if isinstance(target_vals, str): + for cmp_type, cmp_id in cmp_map.items(): + if (cmp_type in target_vals) and (cmp_id != ''): # noqa: PLC1901 + target_vals = target_vals.replace( # noqa: PLW2901 + cmp_type, cmp_id + ) + + new_target_vals = target_vals + + else: + # we assume that target_vals is a list of str + new_target_vals = [] + + for target_val in target_vals: + for cmp_type, cmp_id in cmp_map.items(): + if (cmp_type in target_val) and ( + cmp_id != '' # noqa: PLC1901 + ): + target_val = target_val.replace( # noqa: PLW2901 + cmp_type, cmp_id + ) + + new_target_vals.append(target_val) + + new_dmg_process[new_source][ds_i] = new_target_vals + + dmg_process = new_dmg_process + + # Remove components not present in the asset model + # from the source components of the damage process. + asset_components = set(self.asset.list_unique_component_ids()) + filtered_dmg_process = {} + for key in dmg_process: + component = key.split('_')[1] + if component in asset_components: + filtered_dmg_process[key] = dmg_process[key] + dmg_process = filtered_dmg_process + + elif damage_process_approach == 'User Defined': + if damage_process_file_path is None: + msg = ( + 'When `damage_process_approach` is set to ' + '`User Defined`, a `damage_process_file_path` ' + 'needs to be provided.' + ) + raise ValueError(msg) + + # load the damage process from a file + with Path(damage_process_file_path).open(encoding='utf-8') as f: + dmg_process = json.load(f) + + elif damage_process_approach == 'None': + # no damage process applied for the calculation + dmg_process = None + + else: + self.log.msg( + f'Prescribed Damage Process not recognized: ' + f'`{damage_process_approach}`.' + ) + + # calculate damages + self.damage.calculate( + dmg_process=dmg_process, + scaling_specification=scaling_specification, + ) + + def calculate_loss( + self, + loss_map_approach: str, + occupancy_type: str, + consequence_database: str, + consequence_database_path: str | None = None, + custom_model_dir: str | None = None, + damage_process_approach: str = 'User Defined', + replacement_cost_parameters: dict[str, float | str] | None = None, + replacement_time_parameters: dict[str, float | str] | None = None, + replacement_carbon_parameters: dict[str, float | str] | None = None, + replacement_energy_parameters: dict[str, float | str] | None = None, + loss_map_path: str | None = None, + decision_variables: tuple[str, ...] | None = None, + ) -> tuple[pd.DataFrame, pd.DataFrame]: + """ + Calculate losses. + + Parameters + ---------- + loss_map_approach: str + Approach for the loss map generation. Can be either + `User Defined` or `Automatic`. + occupancy_type: str + Occupancy type. + consequence_database: str + Name of the consequence database. + consequence_database_path: str or None + Optional path to a consequence database file. + custom_model_dir: str or None + Optional directory for custom models. + damage_process_approach: str + Damage process approach. Defaults to `User Defined`. + replacement_cost_parameters: dict or None + Parameters for replacement cost. + replacement_time_parameters: dict or None + Parameters for replacement time. + replacement_carbon_parameters: dict or None + Parameters for replacement carbon. + replacement_energy_parameters: dict or None + Parameters for replacement energy. + loss_map_path: str or None + Optional path to a loss map file. + decision_variables: tuple[str] or None + Optional decision variables for the assessment. + + Returns + ------- + tuple + Dataframe with the aggregated loss of each realization, + and another boolean dataframe with information on which DV + thresholds were exceeded in each realization, triggering + replacement. If no thresholds are specified it only + contains False values. + + Raises + ------ + ValueError + When an invalid loss map approach is specified. + + """ + conseq_df, consequence_db = self.load_consequence_info( + consequence_database, + consequence_database_path, + custom_model_dir, + ) + + # remove duplicates from conseq_df + conseq_df = conseq_df.loc[conseq_df.index.unique(), :] + + # add the replacement consequence to the data + adf = pd.DataFrame( + columns=conseq_df.columns, + index=pd.MultiIndex.from_tuples( + [ + ('replacement', 'Cost'), + ('replacement', 'Time'), + ('replacement', 'Carbon'), + ('replacement', 'Energy'), + ] + ), + ) + + _loss__add_replacement_cost( + adf, + damage_process_approach, + unit=get(replacement_cost_parameters, 'Unit'), + median=get(replacement_cost_parameters, 'Median'), + distribution=get(replacement_cost_parameters, 'Distribution'), + theta_1=get(replacement_cost_parameters, 'Theta_1'), + ) + + _loss__add_replacement_time( + adf, + damage_process_approach, + conseq_df, + occupancy_type=occupancy_type, + unit=get(replacement_time_parameters, 'Unit'), + median=get(replacement_time_parameters, 'Median'), + distribution=get(replacement_time_parameters, 'Distribution'), + theta_1=get(replacement_time_parameters, 'Theta_1'), + ) + + _loss__add_replacement_carbon( + adf, + damage_process_approach, + unit=get(replacement_carbon_parameters, 'Unit'), + median=get(replacement_carbon_parameters, 'Median'), + distribution=get(replacement_carbon_parameters, 'Distribution'), + theta_1=get(replacement_carbon_parameters, 'Theta_1'), + ) + + _loss__add_replacement_energy( + adf, + damage_process_approach, + unit=get(replacement_energy_parameters, 'Unit'), + median=get(replacement_energy_parameters, 'Median'), + distribution=get(replacement_energy_parameters, 'Distribution'), + theta_1=get(replacement_energy_parameters, 'Theta_1'), + ) + + # prepare the loss map + loss_map = None + if loss_map_approach == 'Automatic': + # get the damage sample + loss_map = _loss__map_auto( + self, conseq_df, damage_process_approach, occupancy_type + ) + + elif loss_map_approach == 'User Defined': + assert custom_model_dir is not None + loss_map = _loss__map_user(custom_model_dir, loss_map_path) + + else: + msg = f'Invalid MapApproach value: `{loss_map_approach}`.' + raise ValueError(msg) + + # prepare additional loss map entries, if needed + if 'DMG-collapse' not in loss_map.index: + loss_map.loc['collapse', 'Repair'] = 'replacement' + loss_map.loc['irreparable', 'Repair'] = 'replacement' + + if decision_variables: + self.loss.decision_variables = decision_variables + + self.loss.add_loss_map(loss_map, loss_map_policy=None) + self.loss.load_model_parameters([*consequence_db, adf]) + + self.loss.calculate() + + df_agg, exceedance_bool_df = self.loss.aggregate_losses(future=True) + assert isinstance(df_agg, pd.DataFrame) + assert isinstance(exceedance_bool_df, pd.DataFrame) + return df_agg, exceedance_bool_df + + def load_consequence_info( + self, + consequence_database: str, + consequence_database_path: str | None = None, + custom_model_dir: str | None = None, + ) -> tuple[pd.DataFrame, list[str]]: + """ + Load consequence information for the assessment. + + Parameters + ---------- + consequence_database: str + Name of the consequence database. + consequence_database_path: str or None + Optional path to a consequence database file. + custom_model_dir: str or None + Optional directory for custom models. + + Returns + ------- + tuple[pd.DataFrame, list[str]] + A tuple containing: + - A DataFrame with the consequence data. + - A list of paths to the consequence databases used. + + Raises + ------ + ValueError + With invalid combinations of arguments. + + """ + if consequence_database in default_dbs['repair']: + consequence_db = [ + 'PelicunDefault/' + default_dbs['repair'][consequence_database], + ] + + conseq_df = self.get_default_data( + default_dbs['repair'][consequence_database][:-4] + ) + else: + consequence_db = [] + + conseq_df = pd.DataFrame() + + if consequence_database_path is not None: + if custom_model_dir is None: + msg = ( + 'When `consequence_database_path` is specified, ' + '`custom_model_dir` needs to be specified as well.' + ) + raise ValueError(msg) + + if 'CustomDLDataFolder' in consequence_database_path: + consequence_database_path = consequence_database_path.replace( + 'CustomDLDataFolder', custom_model_dir + ) + + consequence_db += [consequence_database_path] + + extra_conseq_df = file_io.load_data( + consequence_database_path, + unit_conversion_factors=None, + orientation=1, + reindex=False, + ) + assert isinstance(extra_conseq_df, pd.DataFrame) + + if isinstance(conseq_df, pd.DataFrame): + conseq_df = pd.concat([conseq_df, extra_conseq_df]) + else: + conseq_df = extra_conseq_df + + consequence_db = consequence_db[::-1] + + return conseq_df, consequence_db + + +def _add_units(raw_demands: pd.DataFrame, length_unit: str) -> pd.DataFrame: + """ + Add units to demand columns in a DataFrame. + + Parameters + ---------- + raw_demands: pd.DataFrame + The raw demand data to which units will be added. + length_unit: str + The unit of length to be used (e.g., 'in' for inches). + + Returns + ------- + pd.DataFrame + The DataFrame with units added to the appropriate demand columns. + + """ + demands = raw_demands.T + + demands.insert(0, 'Units', np.nan) + + if length_unit == 'in': + length_unit = 'inch' + + demands = pd.DataFrame( + base.convert_to_MultiIndex(demands, axis=0).sort_index(axis=0).T + ) + + nlevels_with_event_id = 4 + dem_level = 1 if demands.columns.nlevels == nlevels_with_event_id else 0 + + # drop demands with no EDP type identified + demands = demands.drop( + demands.columns[demands.columns.get_level_values(dem_level) == ''], + axis=1, + ) + + # assign units + demand_cols = demands.columns.get_level_values(dem_level).to_list() + + # remove additional info from demand names + demand_cols = [d.split('_')[0] for d in demand_cols] + + # acceleration + acc_edps = ['PFA', 'PGA', 'SA'] + edp_mask = np.isin(demand_cols, acc_edps) + + if np.any(edp_mask): + demands.iloc[0, edp_mask] = length_unit + 'ps2' # type: ignore + + # speed + speed_edps = ['PFV', 'PWS', 'PGV', 'SV'] + edp_mask = np.isin(demand_cols, speed_edps) + + if np.any(edp_mask): + demands.iloc[0, edp_mask] = length_unit + 'ps' # type: ignore + + # displacement + disp_edps = ['PFD', 'PIH', 'SD', 'PGD'] + edp_mask = np.isin(demand_cols, disp_edps) + + if np.any(edp_mask): + demands.iloc[0, edp_mask] = length_unit # type: ignore + + # drift ratio + rot_edps = ['PID', 'PRD', 'DWD', 'RDR', 'PMD', 'RID'] + edp_mask = np.isin(demand_cols, rot_edps) + + if np.any(edp_mask): + demands.iloc[0, edp_mask] = 'unitless' # type: ignore + + # convert back to simple header and return the DF + return base.convert_to_SimpleIndex(demands, axis=1) + + +def _loss__add_replacement_energy( + adf: pd.DataFrame, + dl_method: str, + unit: str | None = None, + median: float | None = None, + distribution: str | None = None, + theta_1: float | None = None, +) -> None: + """ + Add replacement energy information. + + Parameters + ---------- + adf : pandas.DataFrame + Dataframe containing loss information. + DL_method : str + Supported methods are 'FEMA P-58'. + unit : str, optional + Unit for the energy value (e.g., 'MJ'). Defaults to None. + median : float, optional + Median replacement energy. If provided, it defines the base + replacement energy value. Defaults to None. + distribution : str, optional + Distribution family to model uncertainty around the median + energy (e.g., 'lognormal'). Required if `median` is + provided. Defaults to None. + theta_1 : float, optional + Distribution parameter (e.g., standard deviation). Required if + `distribution` is provided. Defaults to None. + + Notes + ----- + If `median` is not provided, a default value is assigned based on + the `DL_method`. For 'FEMA P-58', the default replacement energy + value is 0 MJ. For other methods, this consequence is removed + from the dataframe entirely. + """ + ren = ('replacement', 'Energy') + if median is not None: + # TODO(JVM): in this case we need unit (add config parser check) + + adf.loc[ren, ('Quantity', 'Unit')] = '1 EA' + adf.loc[ren, ('DV', 'Unit')] = unit + adf.loc[ren, ('DS1', 'Theta_0')] = median + + if distribution is not None: + # TODO(JVM): in this case we need theta_1 (add config parser check) + + adf.loc[ren, ('DS1', 'Family')] = distribution + adf.loc[ren, ('DS1', 'Theta_1')] = theta_1 + elif dl_method == 'FEMA P-58': + adf.loc[ren, ('Quantity', 'Unit')] = '1 EA' + adf.loc[ren, ('DV', 'Unit')] = 'MJ' + adf.loc[ren, ('DS1', 'Theta_0')] = 0 + + else: + # for everything else, remove this consequence + adf = adf.drop(ren) + + +def _loss__add_replacement_carbon( + adf: pd.DataFrame, + damage_process_approach: str, + unit: str | None = None, + median: float | None = None, + distribution: str | None = None, + theta_1: float | None = None, +) -> None: + """ + Add replacement carbon emission information. + + Parameters + ---------- + adf : pandas.DataFrame + Dataframe containing loss information. + damage_process_approach : str + Supported approaches include 'FEMA P-58'. + unit : str, optional + Unit for the carbon emission value (e.g., 'kg'). Defaults to + None. + median : float, optional + Median replacement carbon emissions. If provided, it defines + the base replacement carbon value. Defaults to None. + distribution : str, optional + Distribution family to model uncertainty around the median + carbon emissions (e.g., 'lognormal'). Required if `median` is + provided. Defaults to None. + theta_1 : float, optional + Distribution parameter (e.g., standard deviation). Required if + `distribution` is provided. Defaults to None. + + Notes + ----- + If `median` is not provided, a default value is assigned based on + the `damage_process_approach`. For 'FEMA P-58', the default + replacement carbon emissions value is 0 kg. For other approaches, + this consequence is removed from the dataframe entirely. + """ + rcarb = ('replacement', 'Carbon') + if median is not None: + # TODO(JVM): in this case we need unit (add config parser check) + + adf.loc[rcarb, ('Quantity', 'Unit')] = '1 EA' + adf.loc[rcarb, ('DV', 'Unit')] = unit + adf.loc[rcarb, ('DS1', 'Theta_0')] = median + + if distribution is not None: + # TODO(JVM): in this case we need theta_1 (add config parser check) + + adf.loc[rcarb, ('DS1', 'Family')] = distribution + adf.loc[rcarb, ('DS1', 'Theta_1')] = theta_1 + elif damage_process_approach == 'FEMA P-58': + adf.loc[rcarb, ('Quantity', 'Unit')] = '1 EA' + adf.loc[rcarb, ('DV', 'Unit')] = 'kg' + adf.loc[rcarb, ('DS1', 'Theta_0')] = 0 + + else: + # for everything else, remove this consequence + adf = adf.drop(rcarb) + + +def _loss__add_replacement_time( + adf: pd.DataFrame, + damage_process_approach: str, + conseq_df: pd.DataFrame, + occupancy_type: str | None = None, + unit: str | None = None, + median: float | None = None, + distribution: str | None = None, + theta_1: float | None = None, +) -> None: + """ + Add replacement time information. + + Parameters + ---------- + adf : pandas.DataFrame + Dataframe containing loss information. + damage_process_approach : str + Supported approaches are 'FEMA P-58', 'Hazus Earthquake - + Buildings'. + conseq_df : pandas.DataFrame + Dataframe containing consequence data for different damage + states. + occupancy_type : str, optional + Type of occupancy, used to look up replacement time in the + consequence dataframe for Hazus Earthquake approach. Defaults + to None. + unit : str, optional + Unit for the replacement time (e.g., 'day, 'worker_day'). + Defaults to None. + median : float, optional + Median replacement time or loss ratio. If provided, it defines + the base replacement time. Defaults to None. + distribution : str, optional + Distribution family to model uncertainty around the median + time (e.g., 'lognormal'). Required if `median` is + provided. Defaults to None. + theta_1 : float, optional + Distribution parameter (e.g., standard deviation). Required if + `distribution` is provided. Defaults to None. + + Notes + ----- + If `median` is not provided, a default value is assigned based on + the `damage_process_approach`. For 'FEMA P-58', the default + replacement time is 0 worker_days. For 'Hazus Earthquake - + Buildings', the replacement time is fetched from `conseq_df` for + the provided `occupancy_type` and corresponds to the total loss + (damage state 5, DS5). In other cases, a placeholder value of 1 is + used. + + """ + rt = ('replacement', 'Time') + if median is not None: + # TODO(JVM): in this case we need unit (add config parser check) + + adf.loc[rt, ('Quantity', 'Unit')] = '1 EA' + adf.loc[rt, ('DV', 'Unit')] = unit + adf.loc[rt, ('DS1', 'Theta_0')] = median + + if distribution is not None: + # TODO(JVM): in this case we need theta_1 (add config parser check) + + adf.loc[rt, ('DS1', 'Family')] = distribution + adf.loc[rt, ('DS1', 'Theta_1')] = theta_1 + elif damage_process_approach == 'FEMA P-58': + adf.loc[rt, ('Quantity', 'Unit')] = '1 EA' + adf.loc[rt, ('DV', 'Unit')] = 'worker_day' + adf.loc[rt, ('DS1', 'Theta_0')] = 0 + + # for Hazus EQ, use 1.0 as a loss_ratio + elif damage_process_approach == 'Hazus Earthquake - Buildings': + adf.loc[rt, ('Quantity', 'Unit')] = '1 EA' + adf.loc[rt, ('DV', 'Unit')] = 'day' + + # load the replacement time that corresponds to total loss + adf.loc[rt, ('DS1', 'Theta_0')] = conseq_df.loc[ + (f'STR.{occupancy_type}', 'Time'), ('DS5', 'Theta_0') + ] + + # otherwise, use 1 (and expect to have it defined by the user) + else: + adf.loc[rt, ('Quantity', 'Unit')] = '1 EA' + adf.loc[rt, ('DV', 'Unit')] = 'loss_ratio' + adf.loc[rt, ('DS1', 'Theta_0')] = 1 + + +def _loss__add_replacement_cost( + adf: pd.DataFrame, + dl_method: str, + unit: str | None = None, + median: float | None = None, + distribution: str | None = None, + theta_1: float | None = None, +) -> None: + """ + Add replacement cost information. + + Parameters + ---------- + adf : pandas.DataFrame + Dataframe containing loss information. + DL_method : str + Supported methods are 'FEMA P-58', 'Hazus Earthquake', and + 'Hazus Hurricane'. + unit : str, optional + Unit for the replacement cost (e.g., 'USD_2011', + 'loss_ratio'). Defaults to None. + median : float, optional + Median replacement cost or loss ratio. If provided, it defines + the base replacement cost. Defaults to None. + distribution : str, optional + Distribution family to model uncertainty around the median + cost (e.g., 'lognormal'). Required if `median` is + provided. Defaults to None. + theta_1 : float, optional + Distribution parameter (e.g., standard deviation). Required if + `distribution` is provided. Defaults to None. + """ + rc = ('replacement', 'Cost') + if median is not None: + # TODO(JVM): in this case we need unit (add config parser check) + + adf.loc[rc, ('Quantity', 'Unit')] = '1 EA' + adf.loc[rc, ('DV', 'Unit')] = unit + adf.loc[rc, ('DS1', 'Theta_0')] = median + + if distribution is not None: + # TODO(JVM): in this case we need theta_1 (add config parser check) + + adf.loc[rc, ('DS1', 'Family')] = distribution + adf.loc[rc, ('DS1', 'Theta_1')] = theta_1 + + elif dl_method == 'FEMA P-58': + adf.loc[rc, ('Quantity', 'Unit')] = '1 EA' + adf.loc[rc, ('DV', 'Unit')] = 'USD_2011' + adf.loc[rc, ('DS1', 'Theta_0')] = 0 + + # for Hazus EQ and HU, use 1.0 as a loss_ratio + elif dl_method in {'Hazus Earthquake', 'Hazus Hurricane'}: + adf.loc[rc, ('Quantity', 'Unit')] = '1 EA' + adf.loc[rc, ('DV', 'Unit')] = 'loss_ratio' + + # store the replacement cost that corresponds to total loss + adf.loc[rc, ('DS1', 'Theta_0')] = 1.00 + + # otherwise, use 1 (and expect to have it defined by the user) + else: + adf.loc[rc, ('Quantity', 'Unit')] = '1 EA' + adf.loc[rc, ('DV', 'Unit')] = 'loss_ratio' + adf.loc[rc, ('DS1', 'Theta_0')] = 1 + + +def _loss__map_user( + custom_model_dir: str, loss_map_path: str | None = None +) -> pd.DataFrame: + """ + Load a user-defined loss map from a specified path. + + Parameters + ---------- + custom_model_dir : str + Directory containing custom models. + loss_map_path : str, optional + Path to the loss map file. The path can include a placeholder + 'CustomDLDataFolder' that will be replaced by + `custom_model_dir`. If not provided, raises a ValueError. + + Returns + ------- + pandas.DataFrame + DataFrame containing the loss map information. + + Raises + ------ + ValueError + If `loss_map_path` is not provided. + + """ + if loss_map_path is not None: + loss_map_path = loss_map_path.replace('CustomDLDataFolder', custom_model_dir) + + else: + msg = 'Missing loss map path.' + raise ValueError(msg) + + return pd.read_csv(loss_map_path, index_col=0) + + +def _loss__map_auto( + assessment: DLCalculationAssessment, + conseq_df: pd.DataFrame, + dl_method: str, + occupancy_type: str | None = None, +) -> pd.DataFrame: + """ + Automatically generate a loss map. + + Automatically generate a loss map based on the damage sample and + the consequence database. + + Parameters + ---------- + assessment : AssessmentBase + The assessment object containing the damage model and sample. + conseq_df : pandas.DataFrame + DataFrame containing consequence data for different damage + states. + DL_method : str + Damage loss method, which defines how the loss map is + generated. Supported methods are 'FEMA P-58', 'Hazus + Earthquake', 'Hazus Hurricane', and 'Hazus Earthquake + Transportation'. + occupancy_type : str, optional + Occupancy type, used to map damage components to the correct + loss models in Hazus Earthquake methods. Defaults to None. + + Returns + ------- + pandas.DataFrame + DataFrame containing the automatically generated loss map, + where the index corresponds to the damage components and the + values indicate the associated loss models. + + Notes + ----- + - For 'FEMA P-58' and 'Hazus Hurricane', the method assumes that + fragility and consequence data have matching component IDs. + - For 'Hazus Earthquake' and 'Hazus Earthquake Transportation', + the method assumes that consequence archetypes are only + differentiated by occupancy type. + + """ + # get the damage sample + dmg_sample = assessment.damage.save_sample() + assert isinstance(dmg_sample, pd.DataFrame) + + # create a mapping for all components that are also in + # the prescribed consequence database + dmg_cmps = dmg_sample.columns.unique(level='cmp') + loss_cmps = conseq_df.index.unique(level=0) + + drivers = [] + loss_models = [] + + if dl_method in {'FEMA P-58', 'Hazus Hurricane'}: + # with these methods, we assume fragility and consequence data + # have the same IDs + + for dmg_cmp in dmg_cmps: + if dmg_cmp == 'collapse': + continue + + if dmg_cmp in loss_cmps: + drivers.append(dmg_cmp) + loss_models.append(dmg_cmp) + + elif dl_method in { + 'Hazus Earthquake', + 'Hazus Earthquake Transportation', + }: + # with Hazus Earthquake we assume that consequence + # archetypes are only differentiated by occupancy type + for dmg_cmp in dmg_cmps: + if dmg_cmp == 'collapse': + continue + + cmp_class = dmg_cmp.split('.')[0] + if occupancy_type is not None: + loss_cmp = f'{cmp_class}.{occupancy_type}' + else: + loss_cmp = cmp_class + + if loss_cmp in loss_cmps: + drivers.append(dmg_cmp) + loss_models.append(loss_cmp) + + return pd.DataFrame(loss_models, columns=['Repair'], index=drivers) + + +class TimeBasedAssessment: + """Time-based assessment.""" diff --git a/pelicun/auto.py b/pelicun/auto.py index 08597b29b..1610a85e6 100644 --- a/pelicun/auto.py +++ b/pelicun/auto.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2023 Leland Stanford Junior University # Copyright (c) 2023 The Regents of the University of California @@ -37,49 +36,51 @@ # Contributors: # Adam Zsarnóczay -""" -This module has classes and methods that auto-populate DL models. -.. rubric:: Contents - -.. autosummary:: - - auto_populate - -""" +"""Classes and methods that auto-populate DL models.""" from __future__ import annotations -import sys + import importlib +import sys from pathlib import Path +from typing import TYPE_CHECKING from pelicun import base +if TYPE_CHECKING: + import pandas as pd + def auto_populate( - config, auto_script_path, **kwargs # pylint: disable=unused-argument -): + config: dict, + auto_script_path: Path, + **kwargs, # noqa: ANN003 +) -> tuple[dict, pd.DataFrame]: """ - Automatically populates the Damage and Loss (DL) configuration for - a Pelicun calculation using predefined rules. + Auto populate the DL configuration with predefined rules. - This function modifies the provided configuration dictionary based - on an external Python script that defines auto-population - rules. It supports using built-in scripts or custom scripts - specified by the user. + Automatically populates the Damage and Loss (DL) configuration for + a Pelicun calculation using predefined rules. This function + modifies the provided configuration dictionary based on an + external Python script that defines auto-population rules. It + supports using built-in scripts or custom scripts specified by the + user. Parameters ---------- - config : dict + config: dict A configuration dictionary with a 'GeneralInformation' key that holds another dictionary with attributes of the asset of interest. This dictionary is modified in-place with auto-populated values. - auto_script_path : str + auto_script_path: str The path pointing to a Python script with the auto-population rules. Built-in scripts can be referenced using the 'PelicunDefault/XY' format where 'XY' is the name of the script. + kwargs + Keyword arguments. Returns ------- @@ -95,33 +96,37 @@ def auto_populate( ValueError If the configuration dictionary does not contain necessary asset information under 'GeneralInformation'. - """ + """ # try to get the AIM attributes - AIM = config.get('GeneralInformation', None) - if AIM is None: - raise ValueError( - "No Asset Information provided for the auto-population routine." - ) + aim = config.get('GeneralInformation') + if aim is None: + msg = 'No Asset Information provided for the auto-population routine.' + raise ValueError(msg) # replace default keyword with actual path in auto_script location - if 'PelicunDefault/' in auto_script_path: - auto_script_path = auto_script_path.replace( - 'PelicunDefault/', f'{base.pelicun_path}/resources/auto/' - ) + path_parts = Path(auto_script_path).resolve().parts + new_parts: list[str] = [ + (Path(base.pelicun_path) / 'resources/auto').resolve().absolute().as_posix() + if part == 'PelicunDefault' + else part + for part in path_parts + ] + if 'PelicunDefault' in path_parts: + auto_script_path = Path(*new_parts) # load the auto population module - ASP = Path(auto_script_path).resolve() - sys.path.insert(0, str(ASP.parent) + '/') - auto_script = importlib.__import__(ASP.name[:-3], globals(), locals(), [], 0) + asp = Path(auto_script_path).resolve() + sys.path.insert(0, str(asp.parent) + '/') + auto_script = importlib.__import__(asp.name[:-3], globals(), locals(), [], 0) auto_populate_ext = auto_script.auto_populate # generate the DL input data - AIM_ap, DL_ap, CMP = auto_populate_ext(AIM=config) + aim_ap, dl_ap, comp = auto_populate_ext(aim=config) # assemble the extended config - config['GeneralInformation'].update(AIM_ap) - config.update({'DL': DL_ap}) + config['GeneralInformation'].update(aim_ap) + config.update({'DL': dl_ap}) # return the extended config data and the component quantities - return config, CMP + return config, comp diff --git a/pelicun/base.py b/pelicun/base.py index d0069f1bf..d26341fb3 100644 --- a/pelicun/base.py +++ b/pelicun/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -38,56 +37,34 @@ # Adam Zsarnóczay # John Vouvakis Manousakis -""" -This module defines constants, basic classes and methods for pelicun. -.. rubric:: Contents - -.. autosummary:: - - load_default_options - update_vals - merge_default_config - convert_to_SimpleIndex - convert_to_MultiIndex - show_matrix - describe - str2bool - float_or_None - int_or_None - process_loc - dedupe_index - dict_raise_on_duplicates - parse_units - convert_units - - Options - Logger - -""" +"""Constants, basic classes, and methods for pelicun.""" from __future__ import annotations -from typing import Any -from typing import TYPE_CHECKING -from collections.abc import Callable -import os -import sys -from datetime import datetime + +import argparse import json +import pprint +import sys +import traceback import warnings +from datetime import datetime, timezone from pathlib import Path -import argparse -import pprint +from typing import TYPE_CHECKING, Any, ClassVar, Optional, TypeVar, overload + +import colorama import numpy as np -from scipy.interpolate import interp1d # type: ignore import pandas as pd -import colorama -from colorama import Fore -from colorama import Style -from pelicun.warnings import PelicunWarning +from colorama import Fore, Style +from scipy.interpolate import interp1d # type: ignore + +from pelicun.pelicun_warnings import PelicunWarning if TYPE_CHECKING: - from pelicun.assessment import Assessment + from collections.abc import Callable + from types import TracebackType + + from pelicun.assessment import AssessmentBase colorama.init() @@ -102,10 +79,12 @@ idx = pd.IndexSlice +T = TypeVar('T') + + class Options: """ - Options objects store analysis options and the logging - configuration. + Analysis options and logging configuration. Attributes ---------- @@ -121,9 +100,10 @@ class Options: value some quantity of a given unit needs to be multiplied to be expressed in the base units). Value specified in the user configuration dictionary. Pelicun comes with a set of default - units which are always loaded (see settings/default_units.json - in the pelicun source code). Units specified in the units_file - overwrite the default units. + units which are always loaded (see + `settings/default_units.json` in the pelicun source + code). Units specified in the units_file overwrite the default + units. demand_offset: dict Demand offsets are used in the process of mapping a component location to its associated EDP. This allows components that @@ -171,51 +151,52 @@ class Options: __slots__ = [ '_asmnt', - 'defaults', - 'sampling_method', - 'list_all_ds', - '_seed', '_rng', - 'units_file', + '_seed', + 'defaults', 'demand_offset', - 'nondir_multi_dict', - 'rho_cost_time', 'eco_scale', + 'eco_scale', + 'error_setup', + 'error_setup', + 'list_all_ds', + 'log', 'log', + 'nondir_multi_dict', + 'rho_cost_time', + 'sampling_method', + 'units_file', ] def __init__( self, user_config_options: dict[str, Any] | None, - assessment: Assessment | None = None, - ): + assessment: AssessmentBase | None = None, + ) -> None: """ - Initializes an Options object. + Initialize an Options object. Parameters ---------- user_config_options: dict, Optional User-specified configuration dictionary. Any provided user_config_options override the defaults. - assessment: Assessment, Optional + assessment: AssessmentBase, Optional Assessment object that will be using this Options object. If it is not intended to use this Options object for an Assessment (e.g. defining an Options object for UQ use), this value should be None. - """ + """ self._asmnt = assessment self.defaults: dict[str, Any] | None = None self.sampling_method: str | None = None self.list_all_ds: bool | None = None - self._seed: float | None = None - - self._rng = np.random.default_rng() merged_config_options = merge_default_config(user_config_options) - self._seed = merged_config_options['Seed'] + self.seed = merged_config_options['Seed'] self.sampling_method = merged_config_options['Sampling']['SamplingMethod'] self.list_all_ds = merged_config_options['ListAllDamageStates'] @@ -226,101 +207,142 @@ def __init__( self.rho_cost_time = merged_config_options['RepairCostAndTimeCorrelation'] self.eco_scale = merged_config_options['EconomiesOfScale'] + self.error_setup = merged_config_options['ErrorSetup'] + # instantiate a Logger object with the finalized configuration self.log = Logger( - merged_config_options['Verbose'], - merged_config_options['LogShowMS'], merged_config_options['LogFile'], - merged_config_options['PrintLog'], + verbose=merged_config_options['Verbose'], + log_show_ms=merged_config_options['LogShowMS'], + print_log=merged_config_options['PrintLog'], ) @property def seed(self) -> float | None: """ - Seed property + Seed property. Returns ------- float Seed value + """ return self._seed @seed.setter def seed(self, value: float) -> None: - """ - seed property setter - """ + """Seed property setter.""" self._seed = value self._rng = np.random.default_rng(self._seed) # type: ignore @property def rng(self) -> np.random.Generator: """ - rng property + rng property. Returns ------- Generator Random generator + """ return self._rng -class Logger: - """ - Logger objects are used to generate log files documenting - execution events and related messages. +# Define a module-level LoggerRegistry +class LoggerRegistry: + """Registry to manage all logger instances.""" + + _loggers: ClassVar[list[Logger]] = [] + + # The @classmethod decorator allows this method to be called on + # the class itself, rather than on instances. It interacts with + # class-level data (like _loggers), enabling a single registry for + # all Logger instances without needing an object of LoggerRegistry + # itself. + @classmethod + def register(cls, logger: Logger) -> None: + """Register a logger instance.""" + cls._loggers.append(logger) + + @classmethod + def log_exception( + cls, + exc_type: type[BaseException], + exc_value: BaseException, + exc_traceback: TracebackType | None, + ) -> None: + """Log exceptions to all registered loggers.""" + message = ( + f"Unhandled exception occurred:" + f"\n" + f"{''.join(traceback.format_exception(exc_type, exc_value, exc_traceback))}" + ) + for logger in cls._loggers: + logger.msg(message) - Attributes - ---------- - verbose: bool - If True, the pelicun echoes more information throughout the - assessment. This can be useful for debugging purposes. The - value is specified in the user's configuration dictionary, - otherwise left as provided in the default configuration file - (see settings/default_config.json in the pelicun source code). - log_show_ms: bool - If True, the timestamps in the log file are in microsecond - precision. The value is specified in the user's configuration - dictionary, otherwise left as provided in the default - configuration file (see settings/default_config.json in the - pelicun source code). - log_file: str, optional - If a value is provided, the log is written to that file. The - value is specified in the user's configuration dictionary, - otherwise left as provided in the default configuration file - (see settings/default_config.json in the pelicun source code). - print_log: bool - If True, the log is also printed to standard output. The - value is specified in the user's configuration dictionary, - otherwise left as provided in the default configuration file - (see settings/default_config.json in the pelicun source code). + # Also call the default excepthook to print the exception to + # the console as is done by default. + sys.__excepthook__(exc_type, exc_value, exc_traceback) - """ + +# Update sys.excepthook to log exceptions in all loggers +# https://docs.python.org/3/library/sys.html#sys.excepthook +sys.excepthook = LoggerRegistry.log_exception + + +class Logger: + """Generate log files documenting execution events.""" __slots__ = [ - 'verbose', - 'log_show_ms', - 'log_file', - 'warning_file', - 'print_log', - 'warning_stack', 'emitted', + 'log_div', + 'log_file', + 'log_show_ms', 'log_time_format', + 'print_log', 'spaces', - 'log_div', + 'verbose', + 'warning_file', + 'warning_stack', ] def __init__( - self, verbose: bool, log_show_ms: bool, log_file: str | None, print_log: bool - ): + self, + log_file: str | None, + *, + verbose: bool, + log_show_ms: bool, + print_log: bool, + ) -> None: """ - Initializes a Logger object. + Initialize a Logger object. Parameters ---------- - see attributes of the Logger class. + verbose: bool + If True, the pelicun echoes more information throughout the + assessment. This can be useful for debugging purposes. The + value is specified in the user's configuration dictionary, + otherwise left as provided in the default configuration file + (see settings/default_config.json in the pelicun source code). + log_show_ms: bool + If True, the timestamps in the log file are in microsecond + precision. The value is specified in the user's configuration + dictionary, otherwise left as provided in the default + configuration file (see settings/default_config.json in the + pelicun source code). + log_file: str, optional + If a value is provided, the log is written to that file. The + value is specified in the user's configuration dictionary, + otherwise left as provided in the default configuration file + (see settings/default_config.json in the pelicun source code). + print_log: bool + If True, the log is also printed to standard output. The + value is specified in the user's configuration dictionary, + otherwise left as provided in the default configuration file + (see settings/default_config.json in the pelicun source code). """ self.verbose = verbose @@ -330,25 +352,16 @@ def __init__( self.log_file = None self.warning_file = None else: - try: - path = Path(log_file) - self.log_file = str(path.resolve()) - name, extension = split_file_name(self.log_file) - self.warning_file = ( - path.parent / (name + '_warnings' + extension) - ).resolve() - with open(self.log_file, 'w', encoding='utf-8') as f: - f.write('') - with open(self.warning_file, 'w', encoding='utf-8') as f: - f.write('') - except BaseException as err: - print( - f"{Fore.RED}WARNING: The filepath provided for the log file " - f"does not point to a valid location: {log_file}. \nPelicun " - f"cannot print the log to a file.\n" - f"The error was: '{err}'{Style.RESET_ALL}" - ) - raise + path = Path(log_file) + self.log_file = str(path.resolve()) + name, extension = split_file_name(self.log_file) + self.warning_file = ( + path.parent / (name + '_warnings' + extension) + ).resolve() + with Path(self.log_file).open('w', encoding='utf-8') as f: + f.write('') + with Path(self.warning_file).open('w', encoding='utf-8') as f: + f.write('') self.print_log = str2bool(print_log) self.warning_stack: list[str] = [] @@ -356,11 +369,12 @@ def __init__( self.reset_log_strings() control_warnings() - def reset_log_strings(self) -> None: - """ - Populates the string-related attributes of the logger - """ + # Register the logger to the LoggerRegistry in order to + # capture raised exceptions. + LoggerRegistry.register(self) + def reset_log_strings(self) -> None: + """Populate the string-related attributes of the logger.""" if self.log_show_ms: self.log_time_format = '%H:%M:%S:%f' # the length of the time string in the log file @@ -375,11 +389,12 @@ def reset_log_strings(self) -> None: def msg( self, msg: str = '', + *, prepend_timestamp: bool = True, prepend_blank_space: bool = True, ) -> None: """ - Writes a message in the log file with the current time as prefix + Write a message in the log file with the current time as prefix. The time is in ISO-8601 format, e.g. 2018-06-16T20:24:04Z @@ -393,35 +408,31 @@ def msg( Controls whether blank space is placed before the message. """ - - # pylint: disable = consider-using-f-string msg_lines = msg.split('\n') for msg_i, msg_line in enumerate(msg_lines): if prepend_timestamp and (msg_i == 0): - formatted_msg = '{} {}'.format( - datetime.now().strftime(self.log_time_format), msg_line + formatted_msg = ( + f'{datetime.now().strftime(self.log_time_format)} {msg_line}' # noqa: DTZ005 ) - elif prepend_timestamp: - formatted_msg = self.spaces + msg_line - elif prepend_blank_space: + elif prepend_timestamp or prepend_blank_space: formatted_msg = self.spaces + msg_line else: formatted_msg = msg_line if self.print_log: - print(formatted_msg) + print(formatted_msg) # noqa: T201 if self.log_file is not None: - with open(self.log_file, 'a', encoding='utf-8') as f: + with Path(self.log_file).open('a', encoding='utf-8') as f: f.write('\n' + formatted_msg) def add_warning(self, msg: str) -> None: """ - Adds a warning to the warning stack. + Add a warning to the warning stack. - Note - ---- + Notes + ----- Warnings are only emitted when `emit_warnings` is called. Parameters @@ -440,15 +451,12 @@ def add_warning(self, msg: str) -> None: self.warning_stack.append(formatted_msg) def emit_warnings(self) -> None: - """ - Issues all warnings and clears the warning stack. - - """ + """Issues all warnings and clears the warning stack.""" for message in self.warning_stack: if message not in self.emitted: - warnings.warn(message, PelicunWarning, stacklevel=2) + warnings.warn(message, PelicunWarning, stacklevel=3) if self.warning_file is not None: - with open(self.warning_file, 'a', encoding='utf-8') as f: + with Path(self.warning_file).open('a', encoding='utf-8') as f: f.write( message.replace(Fore.RED, '') .replace(Style.RESET_ALL, '') @@ -458,7 +466,7 @@ def emit_warnings(self) -> None: self.emitted = self.emitted.union(set(self.warning_stack)) self.warning_stack = [] - def warn(self, msg: str) -> None: + def warning(self, msg: str) -> None: """ Add an emit a warning immediately. @@ -471,28 +479,20 @@ def warn(self, msg: str) -> None: self.add_warning(msg) self.emit_warnings() - def div(self, prepend_timestamp: bool = False) -> None: - """ - Adds a divider line in the log file - """ - - if prepend_timestamp: - msg = self.log_div - else: - msg = '-' * 80 + def div(self, *, prepend_timestamp: bool = False) -> None: + """Add a divider line in the log file.""" + msg = self.log_div if prepend_timestamp else '-' * 80 self.msg(msg, prepend_timestamp=prepend_timestamp) def print_system_info(self) -> None: - """ - Writes system information in the log. - """ - + """Write system information in the log.""" self.msg( 'System Information:', prepend_timestamp=False, prepend_blank_space=False ) + start = datetime.now().strftime('%Y-%m-%dT%H:%M:%S') # noqa: DTZ005 self.msg( - f'local time zone: {datetime.utcnow().astimezone().tzinfo}\n' - f'start time: {datetime.now().strftime("%Y-%m-%dT%H:%M:%S")}\n' + f'local time zone: {datetime.now(timezone.utc).astimezone().tzinfo}\n' + f'start time: {start}\n' f'python: {sys.version}\n' f'numpy: {np.__version__}\n' f'pandas: {pd.__version__}\n', @@ -501,11 +501,13 @@ def print_system_info(self) -> None: # get the absolute path of the pelicun directory -pelicun_path = Path(os.path.dirname(os.path.abspath(__file__))) +pelicun_path = Path(__file__).resolve().parent def split_file_name(file_path: str) -> tuple[str, str]: """ + Separate a file name from the extension. + Separates a file name from the extension accounting for the case where the file name itself contains periods. @@ -531,14 +533,13 @@ def split_file_name(file_path: str) -> tuple[str, str]: def control_warnings() -> None: """ - Convenience function to turn warnings on/off + Turn warnings on/off. See also: `pelicun/pytest.ini`. Devs: make sure to update that file when addressing & eliminating warnings. """ if not sys.warnoptions: - # Here we specify *specific* warnings to ignore. # 'message' -- a regex that the warning message must match @@ -546,48 +547,49 @@ def control_warnings() -> None: # and plan to address them soon. warnings.filterwarnings( - action='ignore', message=".*Use to_numeric without passing `errors`.*" + action='ignore', message='.*Use to_numeric without passing `errors`.*' ) warnings.filterwarnings( action='ignore', message=".*errors='ignore' is deprecated.*" ) warnings.filterwarnings( action='ignore', - message=".*The previous implementation of stack is deprecated.*", + message='.*The previous implementation of stack is deprecated.*', ) warnings.filterwarnings( action='ignore', - message=".*Setting an item of incompatible dtype is deprecated.*", + message='.*Setting an item of incompatible dtype is deprecated.*', ) warnings.filterwarnings( action='ignore', - message=".*DataFrame.groupby with axis=1 is deprecated.*", + message='.*DataFrame.groupby with axis=1 is deprecated.*', ) def load_default_options() -> dict: """ - Load the default_config.json file to set options to default values + Load the default_config.json file to set options to default values. Returns ------- dict Default options - """ - with open( - pelicun_path / "settings/default_config.json", 'r', encoding='utf-8' + """ + with Path(pelicun_path / 'settings/default_config.json').open( + encoding='utf-8' ) as f: default_config = json.load(f) - default_options = default_config['Options'] - return default_options + return default_config['Options'] def update_vals( - update: dict, primary: dict, update_path: str, primary_path: str + update_value: dict, primary: dict, update_path: str, primary_path: str ) -> None: """ + Transfer values between nested dictionaries. + Updates the values of the `update` nested dictionary with those provided in the `primary` nested dictionary. If a key already exists in update, and does not map to another @@ -595,7 +597,7 @@ def update_vals( Parameters ---------- - update: dict + update_value: dict Dictionary -which can contain nested dictionaries- to be updated based on the values of `primary`. New keys existing in `primary` are added to `update`. Values of which keys @@ -616,62 +618,56 @@ def update_vals( If primary[key] is dict but update[key] is not. ValueError If update[key] is dict but primary[key] is not. - """ - - # pylint: disable=else-if-used - # (`consider using elif`) + """ # we go over the keys of `primary` - for key in primary: + for key in primary: # noqa: PLC0206 # if `primary[key]` is a dictionary: if isinstance(primary[key], dict): # if the same `key` does not exist in update, # we associate it with an empty dictionary. - if key not in update: - update[key] = {} + if key not in update_value: + update_value[key] = {} # if it exists already, it should map to # a dictionary. - elif not isinstance(update[key], dict): - raise ValueError( + elif not isinstance(update_value[key], dict): + msg = ( f'{update_path}["{key}"] ' 'should map to a dictionary. ' 'The specified value is ' - f'{update_path}["{key}"] = {update[key]}, but ' + f'{update_path}["{key}"] = {update_value[key]}, but ' f'the default value is ' f'{primary_path}["{key}"] = {primary[key]}. ' f'Please revise {update_path}["{key}"].' ) + raise ValueError(msg) # With both being dictionaries, we use recursion. update_vals( - update[key], + update_value[key], primary[key], f'{update_path}["{key}"]', f'{primary_path}["{key}"]', ) # if `primary[key]` is NOT a dictionary: - else: - # if `key` does not exist in `update`, we add it, with - # its corresponding value. - if key not in update: - update[key] = primary[key] - else: - # key exists in update and should be left alone, - # but we must check that it's not a dict here: - if isinstance(update[key], dict): - raise ValueError( - f'{update_path}["{key}"] ' - 'should not map to a dictionary. ' - f'The specified value is ' - f'{update_path}["{key}"] = {update[key]}, but ' - f'the default value is ' - f'{primary_path}["{key}"] = {primary[key]}. ' - f'Please revise {update_path}["{key}"].' - ) - # pylint: enable=else-if-used + elif key not in update_value: + update_value[key] = primary[key] + elif isinstance(update_value[key], dict): + msg = ( + f'{update_path}["{key}"] ' + 'should not map to a dictionary. ' + f'The specified value is ' + f'{update_path}["{key}"] = {update_value[key]}, but ' + f'the default value is ' + f'{primary_path}["{key}"] = {primary[key]}. ' + f'Please revise {update_path}["{key}"].' + ) + raise ValueError(msg) def merge_default_config(user_config: dict | None) -> dict: """ + Merge default config with user's options. + Merge the user-specified config with the configuration defined in the default_config.json file. If the user-specified config does not include some option available in the default options, then the @@ -686,8 +682,8 @@ def merge_default_config(user_config: dict | None) -> dict: ------- dict Merged configuration dictionary - """ + """ config = user_config # start from the user's config default_config = load_default_options() @@ -702,11 +698,28 @@ def merge_default_config(user_config: dict | None) -> dict: return config +# https://stackoverflow.com/questions/52445559/ +# how-can-i-type-hint-a-function-where-the- +# return-type-depends-on-the-input-type-o + + +@overload def convert_to_SimpleIndex( - data: pd.DataFrame, axis: int = 0, inplace: bool = False -) -> pd.DataFrame: + data: pd.DataFrame, axis: int = 0, *, inplace: bool = False +) -> pd.DataFrame: ... + + +@overload +def convert_to_SimpleIndex( + data: pd.Series, axis: int = 0, *, inplace: bool = False +) -> pd.Series: ... + + +def convert_to_SimpleIndex( # noqa: N802 + data: pd.DataFrame | pd.Series, axis: int = 0, *, inplace: bool = False +) -> pd.DataFrame | pd.Series: """ - Converts the index of a DataFrame to a simple, one-level index + Convert the index of a DataFrame to a simple, one-level index. The target index uses standard SimCenter convention to identify different levels: a dash character ('-') is used to separate each @@ -732,22 +745,19 @@ def convert_to_SimpleIndex( ------ ValueError When an invalid axis parameter is specified - """ + """ if axis in {0, 1}: - if inplace: - data_mod = data - else: - data_mod = data.copy() + data_mod = data if inplace else data.copy() if axis == 0: # only perform this if there are multiple levels if data.index.nlevels > 1: simple_name = '-'.join( - [n if n is not None else "" for n in data.index.names] + [n if n is not None else '' for n in data.index.names] ) simple_index = [ - '-'.join([str(id_i) for id_i in id]) for id in data.index + '-'.join([str(id_i) for id_i in idx]) for idx in data.index ] data_mod.index = pd.Index(simple_index, name=simple_name) @@ -757,26 +767,39 @@ def convert_to_SimpleIndex( # only perform this if there are multiple levels if data.columns.nlevels > 1: simple_name = '-'.join( - [n if n is not None else "" for n in data.columns.names] + [n if n is not None else '' for n in data.columns.names] ) simple_index = [ - '-'.join([str(id_i) for id_i in id]) for id in data.columns + '-'.join([str(id_i) for id_i in idx]) for idx in data.columns ] data_mod.columns = pd.Index(simple_index, name=simple_name) data_mod.columns.name = simple_name else: - raise ValueError(f"Invalid axis parameter: {axis}") + msg = f'Invalid axis parameter: {axis}' + raise ValueError(msg) return data_mod +@overload def convert_to_MultiIndex( - data: pd.DataFrame, axis: int = 0, inplace: bool = False -) -> pd.DataFrame: + data: pd.DataFrame, axis: int = 0, *, inplace: bool = False +) -> pd.DataFrame: ... + + +@overload +def convert_to_MultiIndex( + data: pd.Series, axis: int = 0, *, inplace: bool = False +) -> pd.Series: ... + + +def convert_to_MultiIndex( # noqa: N802 + data: pd.DataFrame | pd.Series, axis: int = 0, *, inplace: bool = False +) -> pd.DataFrame | pd.Series: """ - Converts the index of a DataFrame to a MultiIndex + Convert the index of a DataFrame to a MultiIndex. We assume that the index uses standard SimCenter convention to identify different levels: a dash character ('-') is expected to @@ -802,8 +825,8 @@ def convert_to_MultiIndex( ------ ValueError If an invalid axis is specified. - """ + """ # check if the requested axis is already a MultiIndex if ((axis == 0) and (isinstance(data.index, pd.MultiIndex))) or ( (axis == 1) and (isinstance(data.columns, pd.MultiIndex)) @@ -818,24 +841,20 @@ def convert_to_MultiIndex( index_labels = [str(label).split('-') for label in data.columns] else: - raise ValueError(f"Invalid axis parameter: {axis}") + msg = f'Invalid axis parameter: {axis}' + raise ValueError(msg) max_lbl_len = np.max([len(labels) for labels in index_labels]) for l_i, labels in enumerate(index_labels): if len(labels) != max_lbl_len: - labels += [ - '', - ] * (max_lbl_len - len(labels)) + labels += [''] * (max_lbl_len - len(labels)) # noqa: PLW2901 index_labels[l_i] = labels index_labels_np = np.array(index_labels) if index_labels_np.shape[1] > 1: - if inplace: - data_mod = data - else: - data_mod = data.copy() + data_mod = data if inplace else data.copy() if axis == 0: data_mod.index = pd.MultiIndex.from_arrays(index_labels_np.T) @@ -850,9 +869,10 @@ def convert_to_MultiIndex( def convert_dtypes(dataframe: pd.DataFrame) -> pd.DataFrame: """ - Convert columns to a numeric datatype whenever possible. The - function replaces None with NA otherwise columns containing None - would continue to have the `object` type + Convert columns to a numeric datatype whenever possible. + + The function replaces None with NA otherwise columns containing + None would continue to have the `object` type. Parameters ---------- @@ -865,7 +885,11 @@ def convert_dtypes(dataframe: pd.DataFrame) -> pd.DataFrame: The modified DataFrame. """ - dataframe.fillna(value=np.nan, inplace=True) + with ( + pd.option_context('future.no_silent_downcasting', True), # noqa: FBT003 + pd.option_context('mode.copy_on_write', True), # noqa: FBT003 + ): + dataframe = dataframe.fillna(value=np.nan).infer_objects() # note: `axis=0` applies the function to the columns # note: ignoring errors is a bad idea and should never be done. In # this case, however, that's not what we do, despite the name of @@ -875,20 +899,23 @@ def convert_dtypes(dataframe: pd.DataFrame) -> pd.DataFrame: # See: # https://pandas.pydata.org/docs/reference/api/pandas.to_numeric.html return dataframe.apply( - lambda x: pd.to_numeric(x, errors='ignore'), axis=0 # type:ignore + lambda x: pd.to_numeric(x, errors='ignore'), # type:ignore + axis=0, ) -def show_matrix(data, use_describe=False): +def show_matrix( + data: np.ndarray | pd.DataFrame, *, use_describe: bool = False +) -> None: """ Print a matrix in a nice way using a DataFrame. Parameters ---------- - data : array-like + data: array-like The matrix data to display. Can be any array-like structure that pandas can convert to a DataFrame. - use_describe : bool, default: False + use_describe: bool, default: False If True, provides a descriptive statistical summary of the matrix including specified percentiles. If False, simply prints the matrix as is. @@ -907,10 +934,13 @@ def multiply_factor_multiple_levels( conditions: dict, factor: float, axis: int = 0, + *, raise_missing: bool = True, ) -> None: """ - Multiply a value to selected rows of a DataFrame that is indexed + Multiply a value to selected rows, in place. + + Multiplies a value to selected rows of a DataFrame that is indexed with a hierarchical index (pd.MultiIndex). The change is done in place. @@ -942,23 +972,23 @@ def multiply_factor_multiple_levels( is True. """ - if axis == 0: idx_to_use = df.index elif axis == 1: idx_to_use = df.columns else: - raise ValueError(f'Invalid axis: `{axis}`') + msg = f'Invalid axis: `{axis}`' + raise ValueError(msg) - mask = pd.Series(True, index=idx_to_use) + mask = pd.Series(data=True, index=idx_to_use) # Apply each condition to update the mask for level, value in conditions.items(): mask &= idx_to_use.get_level_values(level) == value - # pylint: disable=singleton-comparison - if np.all(mask == False) and raise_missing: # noqa - raise ValueError(f'No rows found matching the conditions: `{conditions}`') + if np.all(mask == False) and raise_missing: # noqa: E712 + msg = f'No rows found matching the conditions: `{conditions}`' + raise ValueError(msg) if axis == 0: df.iloc[mask.to_numpy()] *= factor @@ -971,10 +1001,12 @@ def _warning( category: type[Warning], filename: str, lineno: int, - file: Any = None, - line: Any = None, + file: Any = None, # noqa: ARG001, ANN401 + line: Any = None, # noqa: ARG001, ANN401 ) -> None: """ + Display warnings in a custom format. + Custom warning function to format and print warnings more attractively. This function modifies how warning messages are displayed, emphasizing the file path and line number from where @@ -982,22 +1014,23 @@ def _warning( Parameters ---------- - message : str + message: str The warning message to be displayed. - category : Warning + category: Warning The category of the warning (unused, but required for compatibility with standard warning signature). - filename : str + filename: str The path of the file from which the warning is issued. The function simplifies the path for display. - lineno : int + lineno: int The line number in the file at which the warning is issued. - file : file-like object, optional + file: file-like object, optional The target file object to write the warning to (unused, but required for compatibility with standard warning signature). - line : str, optional + line: str, optional Line of code causing the warning (unused, but required for compatibility with standard warning signature). + """ # pylint:disable = unused-argument if category != PelicunWarning: @@ -1008,21 +1041,18 @@ def _warning( else: file_path = None - if file_path is not None: - python_file = '/'.join(file_path[-3:]) - else: - python_file = filename - print(f'WARNING in {python_file} at line {lineno}\n{message}\n') + python_file = '/'.join(file_path[-3:]) if file_path is not None else filename + print(f'WARNING in {python_file} at line {lineno}\n{message}\n') # noqa: T201 else: - print(message) + print(message) # noqa: T201 warnings.showwarning = _warning # type: ignore def describe( - df, - percentiles=( + data: pd.DataFrame | pd.Series | np.ndarray, + percentiles: tuple[float, ...] = ( 0.001, 0.023, 0.10, @@ -1033,11 +1063,12 @@ def describe( 0.977, 0.999, ), -): +) -> pd.DataFrame: """ + Extend descriptive statistics. + Provides extended descriptive statistics for given data, including percentiles and log standard deviation for applicable columns. - This function accepts both pandas Series and DataFrame objects directly, or any array-like structure which can be converted to them. It calculates common descriptive statistics and optionally @@ -1046,10 +1077,10 @@ def describe( Parameters ---------- - df : pd.Series, pd.DataFrame, or array-like + data: pd.Series, pd.DataFrame, or array-like The data to describe. If array-like, it is converted to a DataFrame or Series before analysis. - percentiles : tuple of float, optional + percentiles: tuple of float, optional Specific percentiles to include in the output. Default includes an extensive range tailored to provide a detailed summary. @@ -1059,37 +1090,37 @@ def describe( pd.DataFrame A DataFrame containing the descriptive statistics of the input data, transposed so that each descriptive statistic is a row. + """ - if not isinstance(df, (pd.Series, pd.DataFrame)): - vals = df - cols = np.arange(vals.shape[1]) if vals.ndim > 1 else 0 + if isinstance(data, np.ndarray): + vals = data if vals.ndim == 1: - df = pd.Series(vals, name=cols) + data = pd.Series(vals, name=0) else: - df = pd.DataFrame(vals, columns=cols) + cols = np.arange(vals.shape[1]) + data = pd.DataFrame(vals, columns=cols) - # convert Series into a DataFrame - if isinstance(df, pd.Series): - df = pd.DataFrame(df) + # convert Series to a DataFrame + if isinstance(data, pd.Series): + data = pd.DataFrame(data) - desc = df.describe(list(percentiles)).T + desc = pd.DataFrame(data.describe(list(percentiles)).T) # add log standard deviation to the stats - desc.insert(3, "log_std", np.nan) + desc.insert(3, 'log_std', np.nan) desc = desc.T for col in desc.columns: - if np.min(df[col]) > 0.0: - desc.loc['log_std', col] = np.std(np.log(df[col]), ddof=1) + if np.min(data[col]) > 0.0: + desc.loc['log_std', col] = np.std(np.log(data[col]), ddof=1) return desc -def str2bool(v: str | bool) -> bool: +def str2bool(v: str | bool) -> bool: # noqa: FBT001 """ - Converts a string representation of truth to boolean True or - False. + Convert a string representation of truth to boolean True or False. This function is designed to convert string inputs that represent boolean values into actual Python boolean types. It handles @@ -1098,7 +1129,7 @@ def str2bool(v: str | bool) -> bool: Parameters ---------- - v : str or bool + v: str or bool The value to convert into a boolean. This can be a boolean itself (in which case it is simply returned) or a string that is expected to represent a boolean value. @@ -1114,6 +1145,7 @@ def str2bool(v: str | bool) -> bool: If `v` is a string that does not correspond to a boolean value, an error is raised indicating that a boolean value was expected. + """ # courtesy of Maxim @ Stackoverflow @@ -1123,13 +1155,13 @@ def str2bool(v: str | bool) -> bool: return True if v.lower() in {'no', 'false', 'False', 'f', 'n', '0'}: return False - raise argparse.ArgumentTypeError('Boolean value expected.') + msg = 'Boolean value expected.' + raise argparse.ArgumentTypeError(msg) -def float_or_None(string: str) -> float | None: +def float_or_None(string: str) -> float | None: # noqa: N802 """ - This is a convenience function for converting strings to float or - None + Convert strings to float or None. Parameters ---------- @@ -1141,18 +1173,17 @@ def float_or_None(string: str) -> float | None: float or None A float, if the given string can be converted to a float. Otherwise, it returns None + """ try: - res = float(string) - return res + return float(string) except ValueError: return None -def int_or_None(string: str) -> int | None: +def int_or_None(string: str) -> int | None: # noqa: N802 """ - This is a convenience function for converting strings to int or - None + Convert strings to int or None. Parameters ---------- @@ -1164,30 +1195,27 @@ def int_or_None(string: str) -> int | None: int or None An int, if the given string can be converted to an int. Otherwise, it returns None + """ try: - res = int(string) - return res + return int(string) except ValueError: return None -def with_parsed_str_na_values(df: pd.DataFrame) -> pd.DataFrame: +def check_if_str_is_na(string: Any) -> bool: # noqa: ANN401 """ - Given a dataframe, this function identifies values that have - string type and can be interpreted as N/A, and replaces them with - actual NA's. + Check if the provided string can be interpreted as N/A. Parameters ---------- - df: pd.DataFrame - Dataframe to process + string: object + The string to evaluate Returns ------- - pd.DataFrame - The dataframe with proper N/A values. - + bool + The evaluation result. Yes, if the string is considered N/A. """ na_vals = { '', @@ -1212,16 +1240,37 @@ def with_parsed_str_na_values(df: pd.DataFrame) -> pd.DataFrame: } # obtained from Pandas' internal STR_NA_VALUES variable. + return isinstance(string, str) and string in na_vals + + +def with_parsed_str_na_values(df: pd.DataFrame) -> pd.DataFrame: + """ + Identify string values interpretable as N/A. + + Given a dataframe, this function identifies values that have + string type and can be interpreted as N/A, and replaces them with + actual NA's. + + Parameters + ---------- + df: pd.DataFrame + Dataframe to process + + Returns + ------- + pd.DataFrame + The dataframe with proper N/A values. + """ # Replace string NA values with actual NaNs return df.apply( - lambda col: col.map( - lambda x: np.nan if isinstance(x, str) and x in na_vals else x - ) + lambda col: col.map(lambda x: np.nan if check_if_str_is_na(x) else x) ) -def dedupe_index(dataframe: pd.DataFrame, dtype: type = str) -> None: +def dedupe_index(dataframe: pd.DataFrame, dtype: type = str) -> pd.DataFrame: """ + Add a `uid` level to the index. + Modifies the index of a DataFrame to ensure all index elements are unique by adding an extra level. Assumes that the DataFrame's original index is a MultiIndex with specified names. A unique @@ -1231,23 +1280,24 @@ def dedupe_index(dataframe: pd.DataFrame, dtype: type = str) -> None: Parameters ---------- - dataframe : pd.DataFrame + dataframe: pd.DataFrame The DataFrame whose index is to be modified. It must have a MultiIndex. - dtype : type, optional + dtype: type, optional The data type for the new index level 'uid'. Defaults to str. - Notes - ----- - This function changes the DataFrame in place, hence it does not - return the DataFrame but modifies the original one provided. + Returns + ------- + dataframe: pd.DataFrame + The original dataframe with an additional `uid` level at the + index. """ inames = dataframe.index.names - dataframe.reset_index(inplace=True) + dataframe = dataframe.reset_index() dataframe['uid'] = (dataframe.groupby([*inames]).cumcount()).astype(dtype) - dataframe.set_index([*inames] + ['uid'], inplace=True) - dataframe.sort_index(inplace=True) + dataframe = dataframe.set_index([*inames, 'uid']) + return dataframe.sort_index() # Input specs @@ -1292,16 +1342,17 @@ def dedupe_index(dataframe: pd.DataFrame, dtype: type = str) -> None: def dict_raise_on_duplicates(ordered_pairs: list[tuple]) -> dict: """ + Construct a dictionary from a list of key-value pairs. + Constructs a dictionary from a list of key-value pairs, raising an exception if duplicate keys are found. - This function ensures that no two pairs have the same key. It is particularly useful when parsing JSON-like data where unique keys are expected but not enforced by standard parsing methods. Parameters ---------- - ordered_pairs : list of tuples + ordered_pairs: list of tuples A list of tuples, each containing a key and a value. Keys are expected to be unique across the list. @@ -1328,18 +1379,19 @@ def dict_raise_on_duplicates(ordered_pairs: list[tuple]) -> dict: ----- This implementation is useful for contexts in which data integrity is crucial and key uniqueness must be ensured. - """ + """ d = {} for k, v in ordered_pairs: if k in d: - raise ValueError(f"duplicate key: {k}") + msg = f'duplicate key: {k}' + raise ValueError(msg) d[k] = v return d -def parse_units( - custom_file: str | None = None, preserve_categories: bool = False +def parse_units( # noqa: C901 + custom_file: str | None = None, *, preserve_categories: bool = False ) -> dict: """ Parse the unit conversion factor JSON file and return a dictionary. @@ -1349,6 +1401,12 @@ def parse_units( custom_file: str, optional If a custom file is provided, only the units specified in the custom file are used. + preserve_categories: bool, optional + If True, maintains the original data types of category + values from the JSON file. If False, converts all values + to floats and flattens the dictionary structure, ensuring + that each unit name is globally unique across categories. + Returns ------- @@ -1360,20 +1418,12 @@ def parse_units( `preserve_categories` is False, the dictionary is flattened to have globally unique unit names. - Raises - ------ - KeyError - If a key is defined twice. - ValueError - If a unit conversion factor is not a float. - FileNotFoundError - If a file does not exist. - Exception - If a file does not have the JSON format. """ - def get_contents(file_path, preserve_categories=False): + def get_contents(file_path: Path, *, preserve_categories: bool = False) -> dict: # noqa: C901 """ + Map unit names to conversion factors. + Parses a unit conversion factors JSON file and returns a dictionary mapping unit names to conversion factors. @@ -1386,10 +1436,10 @@ def get_contents(file_path, preserve_categories=False): Parameters ---------- - file_path : str + file_path: str The file path to a JSON file containing unit conversion factors. If not provided, a default file is used. - preserve_categories : bool, optional + preserve_categories: bool, optional If True, maintains the original data types of category values from the JSON file. If False, converts all values to floats and flattens the dictionary structure, ensuring @@ -1408,10 +1458,9 @@ def get_contents(file_path, preserve_categories=False): FileNotFoundError If the specified file does not exist. ValueError - If a unit name is duplicated, a conversion factor is not a - float, or other JSON structure issues are present. - json.decoder.JSONDecodeError - If the file is not a valid JSON file. + If a unit name is duplicated or other JSON structure issues are present. + TypeError + If a conversion factor is not a float. TypeError If any value that needs to be converted to float cannot be converted. @@ -1423,30 +1472,35 @@ def get_contents(file_path, preserve_categories=False): >>> parse_units('custom_units.json', preserve_categories=True) { 'Length': {'m': 1.0, 'cm': 0.01, 'mm': 0.001} } + """ try: - with open(file_path, 'r', encoding='utf-8') as f: + with Path(file_path).open(encoding='utf-8') as f: dictionary = json.load(f, object_pairs_hook=dict_raise_on_duplicates) except FileNotFoundError as exc: - raise FileNotFoundError(f'{file_path} was not found.') from exc + msg = f'{file_path} was not found.' + raise FileNotFoundError(msg) from exc except json.decoder.JSONDecodeError as exc: - raise ValueError(f'{file_path} is not a valid JSON file.') from exc + msg = f'{file_path} is not a valid JSON file.' + raise ValueError(msg) from exc for category_dict in list(dictionary.values()): # ensure all first-level keys point to a dictionary if not isinstance(category_dict, dict): - raise ValueError( + msg = ( f'{file_path} contains first-level keys ' - 'that don\'t point to a dictionary' + "that don't point to a dictionary" ) + raise TypeError(msg) # convert values to float - for key, val in category_dict.items(): - try: + try: + for key, val in category_dict.items(): category_dict[key] = float(val) - except (ValueError, TypeError) as exc: - raise type(exc)( - f'Unit {key} has a value of {val} ' - 'which cannot be interpreted as a float' - ) from exc + except (ValueError, TypeError) as exc: + msg = ( + f'Unit {key} has a value of {val} ' + 'which cannot be interpreted as a float' + ) + raise type(exc)(msg) from exc if preserve_categories: return dictionary @@ -1455,27 +1509,31 @@ def get_contents(file_path, preserve_categories=False): for category in dictionary: for unit_name, factor in dictionary[category].items(): if unit_name in flattened: - raise ValueError(f'{unit_name} defined twice in {file_path}.') + msg = f'{unit_name} defined twice in {file_path}.' + raise ValueError(msg) flattened[unit_name] = factor return flattened if custom_file: - return get_contents(custom_file, preserve_categories) + return get_contents( + Path(custom_file), preserve_categories=preserve_categories + ) return get_contents( - pelicun_path / "settings/default_units.json", preserve_categories + pelicun_path / 'settings/default_units.json', + preserve_categories=preserve_categories, ) -def convert_units( +def convert_units( # noqa: C901 values: float | list[float] | np.ndarray, unit: str, to_unit: str, category: str | None = None, ) -> float | list[float] | np.ndarray: """ - Converts numeric values between different units. + Convert numeric values between different units. Supports conversion within a specified category of units and automatically infers the category if not explicitly provided. It @@ -1483,13 +1541,13 @@ def convert_units( Parameters ---------- - values (float | list[float] | np.ndarray): + values: (float | list[float] | np.ndarray) The numeric value(s) to convert. - unit (str): + unit: (str) The current unit of the values. - to_unit (str): + to_unit: (str) The target unit to convert the values into. - category (Optional[str]): + category: (Optional[str]) The category of the units (e.g., 'length', 'pressure'). If not provided, the category will be inferred based on the provided units. @@ -1510,13 +1568,13 @@ def convert_units( and `to_unit` are not in the same category. """ - if isinstance(values, (float, list)): vals = np.atleast_1d(values) elif isinstance(values, np.ndarray): vals = values else: - raise TypeError('Invalid input type for `values`') + msg = 'Invalid input type for `values`' + raise TypeError(msg) # load default units all_units = parse_units(preserve_categories=True) @@ -1524,11 +1582,13 @@ def convert_units( # if a category is given use it, otherwise try to determine it if category: if category not in all_units: - raise ValueError(f'Unknown category: `{category}`') + msg = f'Unknown category: `{category}`' + raise ValueError(msg) units = all_units[category] for unt in unit, to_unit: if unt not in units: - raise ValueError(f'Unknown unit: `{unt}`') + msg = f'Unknown unit: `{unt}`' + raise ValueError(msg) else: unit_category: str | None = None for key in all_units: @@ -1537,13 +1597,15 @@ def convert_units( unit_category = key break if not unit_category: - raise ValueError(f'Unknown unit `{unit}`') + msg = f'Unknown unit `{unit}`' + raise ValueError(msg) units = all_units[unit_category] if to_unit not in units: - raise ValueError( + msg = ( f'`{unit}` is a `{unit_category}` unit, but `{to_unit}` ' f'is not specified in that category.' ) + raise ValueError(msg) # convert units from_factor = units[unit] @@ -1563,6 +1625,8 @@ def stringterpolation( arguments: str, ) -> Callable[[np.ndarray], np.ndarray]: """ + Linear interpolation from strings. + Turns a string of specially formatted arguments into a multilinear interpolating function. @@ -1595,7 +1659,7 @@ def invert_mapping(original_dict: dict) -> dict: Parameters ---------- - original_dict : dict + original_dict: dict Dictionary with values that are lists of hashable items. Returns @@ -1616,6 +1680,213 @@ def invert_mapping(original_dict: dict) -> dict: for key, value_list in original_dict.items(): for value in value_list: if value in inverted_dict: - raise ValueError('Cannot invert mapping with duplicate values.') + msg = 'Cannot invert mapping with duplicate values.' + raise ValueError(msg) inverted_dict[value] = key return inverted_dict + + +def get( + d: dict | None, + path: str, + default: Any | None = None, # noqa: ANN401 +) -> Any: # noqa: ANN401 + """ + Path-like dictionary value retrieval. + + Retrieves a value from a nested dictionary using a path with '/' + as the separator. + + Parameters + ---------- + d: dict + The dictionary to search. + path: str + The path to the desired value, with keys separated by '/'. + default: Any, optional + The value to return if the path is not found. Defaults to + None. + + Returns + ------- + Any + The value found at the specified path, or the default value if + the path is not found. + + Examples + -------- + >>> config = { + ... "DL": { + ... "Outputs": { + ... "Format": { + ... "JSON": "desired_value" + ... } + ... } + ... } + ... } + >>> get(config, '/DL/Outputs/Format/JSON', default='default_value') + 'desired_value' + >>> get(config, '/DL/Outputs/Format/XML', default='default_value') + 'default_value' + + """ + if d is None: + return default + keys = path.strip('/').split('/') + current_dict = d + try: + for key in keys: + current_dict = current_dict[key] + return current_dict # noqa: TRY300 + except (KeyError, TypeError): + return default + + +def update( + d: dict[str, Any], + path: str, + value: Any, # noqa: ANN401 + *, + only_if_empty_or_none: bool = False, +) -> None: + """ + Set a value in a nested dictionary using a path with '/' as the separator. + + Parameters + ---------- + d: dict + The dictionary to update. + path: str + The path to the desired value, with keys separated by '/'. + value: Any + The value to set at the specified path. + only_if_empty_or_none: bool, optional + If True, only update the value if it is None or an empty + dictionary. Defaults to False. + + Examples + -------- + >>> d = {} + >>> update(d, 'x/y/z', 1) + >>> d + {'x': {'y': {'z': 1}}} + + >>> update(d, 'x/y/z', 2, only_if_empty_or_none=True) + >>> d + {'x': {'y': {'z': 1}}} # value remains 1 since it is not empty or None + + >>> update(d, 'x/y/z', 2) + >>> d + {'x': {'y': {'z': 2}}} # value is updated to 2 + + """ + keys = path.strip('/').split('/') + current_dict = d + for key in keys[:-1]: + if key not in current_dict or not isinstance(current_dict[key], dict): + current_dict[key] = {} + current_dict = current_dict[key] + if only_if_empty_or_none: + if is_unspecified(current_dict, keys[-1]): + current_dict[keys[-1]] = value + else: + current_dict[keys[-1]] = value + + +def is_unspecified(d: dict[str, Any], path: str) -> bool: + """ + Check if something is specified. + + Checks if a value in a nested dictionary is either non-existent, + None, NaN, or an empty dictionary or list. + + Parameters + ---------- + d: dict + The dictionary to search. + path: str + The path to the desired value, with keys separated by '/'. + + Returns + ------- + bool + True if the value is non-existent, None, or an empty + dictionary or list. False otherwise. + + Examples + -------- + >>> config = { + ... "DL": { + ... "Outputs": { + ... "Format": { + ... "JSON": "desired_value", + ... "EmptyDict": {} + ... } + ... } + ... } + ... } + >>> is_unspecified(config, '/DL/Outputs/Format/JSON') + False + >>> is_unspecified(config, '/DL/Outputs/Format/XML') + True + >>> is_unspecified(config, '/DL/Outputs/Format/EmptyDict') + True + + """ + value = get(d, path, default=None) + if value is None: + return True + if pd.isna(value): + return True + if value == {}: + return True + return value == [] + + +def is_specified(d: dict[str, Any], path: str) -> bool: + """ + Opposite of `is_unspecified()`. + + Parameters + ---------- + d: dict + The dictionary to search. + path: str + The path to the desired value, with keys separated by '/'. + + Returns + ------- + bool + True if the value is specified, False otherwise. + + """ + return not is_unspecified(d, path) + + +def ensure_value(value: T | None) -> T: + """ + Ensure a variable is not None. + + This function checks that the provided variable is not None. It is + used to assist with type hinting by avoiding repetitive `assert + value is not None` statements throughout the code. + + Parameters + ---------- + value : Optional[T] + The variable to check, which can be of any type or None. + + Returns + ------- + T + The same variable, guaranteed to be non-None. + + Raises + ------ + TypeError + If the provided variable is None. + + """ + if value is None: + raise TypeError + return value diff --git a/pelicun/db.py b/pelicun/db.py deleted file mode 100644 index bcf875b1c..000000000 --- a/pelicun/db.py +++ /dev/null @@ -1,2885 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (c) 2018-2022 Leland Stanford Junior University -# Copyright (c) 2018-2022 The Regents of the University of California -# -# This file is part of pelicun. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its contributors -# may be used to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -# You should have received a copy of the BSD 3-Clause License along with -# pelicun. If not, see . -# -# Contributors: -# Adam Zsarnóczay - -""" -This module has classes and methods to manage databases used by pelicun. - -.. rubric:: Contents - -.. autosummary:: - - create_FEMA_P58_fragility_db - create_FEMA_P58_repair_db - create_FEMA_P58_bldg_redtag_db - - create_Hazus_EQ_fragility_db - create_Hazus_EQ_repair_db - -""" - -from __future__ import annotations -import re -import json -from pathlib import Path -from copy import deepcopy -import numpy as np -from scipy.stats import norm # type: ignore -import pandas as pd - -from pelicun import base -from pelicun.uq import fit_distribution_to_percentiles - -idx = base.idx - - -# pylint: disable=too-many-statements -# pylint: disable=too-many-locals - - -def parse_DS_Hierarchy(DSH): - """ - Parses the FEMA P58 DS hierarchy into a set of arrays. - - Parameters - ---------- - DSH: str - Damage state hierarchy - - Returns - ------- - list - Damage state setup - """ - if DSH[:3] == 'Seq': - DSH = DSH[4:-1] - - DS_setup = [] - - while len(DSH) > 0: - if DSH[:2] == 'DS': - DS_setup.append(DSH[:3]) - DSH = DSH[4:] - elif DSH[:5] in {'MutEx', 'Simul'}: - closing_pos = DSH.find(')') - subDSH = DSH[: closing_pos + 1] - DSH = DSH[closing_pos + 2 :] - - DS_setup.append([subDSH[:5]] + subDSH[6:-1].split(',')) - - return DS_setup - - -def create_FEMA_P58_fragility_db( - source_file, - meta_file='', - target_data_file='damage_DB_FEMA_P58_2nd.csv', - target_meta_file='damage_DB_FEMA_P58_2nd.json', -): - """ - Create a fragility parameter database based on the FEMA P58 data - - The method was developed to process v3.1.2 of the FragilityDatabase xls - that is provided with FEMA P58 2nd edition. - - Parameters - ---------- - source_file: string - Path to the fragility database file. - meta_file: string - Path to the JSON file with metadata about the database. - target_data_file: string - Path where the fragility data file should be saved. A csv file is - expected. - target_meta_file: string - Path where the fragility metadata should be saved. A json file is - expected. - - Raises - ------ - ValueError - If there are problems with the mutually exclusive damage state - definition of some component. - """ - - # parse the source file - df = pd.read_excel( - source_file, - sheet_name='Summary', - header=2, - index_col=1, - true_values=["YES", "Yes", "yes"], - false_values=["NO", "No", "no"], - ) - - # parse the extra metadata file - if Path(meta_file).is_file(): - with open(meta_file, 'r', encoding='utf-8') as f: - frag_meta = json.load(f) - else: - frag_meta = {} - - # remove the empty rows and columns - df.dropna(axis=0, how='all', inplace=True) - df.dropna(axis=1, how='all', inplace=True) - - # filter the columns that we need for the fragility database - cols_to_db = [ - "Demand Parameter (value):", - "Demand Parameter (unit):", - "Demand Location (use floor above? Yes/No)", - "Directional?", - "DS Hierarchy", - "DS 1, Probability", - "DS 1, Median Demand", - "DS 1, Total Dispersion (Beta)", - "DS 2, Probability", - "DS 2, Median Demand", - "DS 2, Total Dispersion (Beta)", - "DS 3, Probability", - "DS 3, Median Demand", - "DS 3, Total Dispersion (Beta)", - "DS 4, Probability", - "DS 4, Median Demand", - "DS 4, Total Dispersion (Beta)", - "DS 5, Probability", - "DS 5, Median Demand", - "DS 5, Total Dispersion (Beta)", - ] - - # filter the columns that we need for the metadata - cols_to_meta = [ - "Component Name", - "Component Description", - "Construction Quality:", - "Seismic Installation Conditions:", - "Comments / Notes", - "Author", - "Fragility Unit of Measure", - "Round to Integer Unit?", - "DS 1, Description", - "DS 1, Repair Description", - "DS 2, Description", - "DS 2, Repair Description", - "DS 3, Description", - "DS 3, Repair Description", - "DS 4, Description", - "DS 4, Repair Description", - "DS 5, Description", - "DS 5, Repair Description", - ] - - # remove special characters to make it easier to work with column names - str_map = { - ord(' '): "_", - ord(':'): None, - ord('('): None, - ord(')'): None, - ord('?'): None, - ord('/'): None, - ord(','): None, - } - - df_db_source = df.loc[:, cols_to_db] - df_db_source.columns = [s.translate(str_map) for s in cols_to_db] - df_db_source.sort_index(inplace=True) - - df_meta = df.loc[:, cols_to_meta] - df_meta.columns = [s.translate(str_map) for s in cols_to_meta] - # replace missing values with an empty string - df_meta.fillna('', inplace=True) - # the metadata shall be stored in strings - df_meta = df_meta.astype(str) - - # initialize the output fragility table - df_db = pd.DataFrame( - columns=[ - "Index", - "Incomplete", - "Demand-Type", - "Demand-Unit", - "Demand-Offset", - "Demand-Directional", - "LS1-Family", - "LS1-Theta_0", - "LS1-Theta_1", - "LS1-DamageStateWeights", - "LS2-Family", - "LS2-Theta_0", - "LS2-Theta_1", - "LS2-DamageStateWeights", - "LS3-Family", - "LS3-Theta_0", - "LS3-Theta_1", - "LS3-DamageStateWeights", - "LS4-Family", - "LS4-Theta_0", - "LS4-Theta_1", - "LS4-DamageStateWeights", - ], - index=df_db_source.index, - dtype=float, - ) - - # initialize the dictionary that stores the fragility metadata - meta_dict = {} - - # add the general information to the meta dict - if "_GeneralInformation" in frag_meta.keys(): - frag_meta = frag_meta["_GeneralInformation"] - - # remove the decision variable part from the general info - frag_meta.pop("DecisionVariables", None) - - meta_dict.update({"_GeneralInformation": frag_meta}) - - # conversion dictionary for demand types - convert_demand_type = { - 'Story Drift Ratio': "Peak Interstory Drift Ratio", - 'Link Rotation Angle': "Peak Link Rotation Angle", - 'Effective Drift': "Peak Effective Drift Ratio", - 'Link Beam Chord Rotation': "Peak Link Beam Chord Rotation", - 'Peak Floor Acceleration': "Peak Floor Acceleration", - 'Peak Floor Velocity': "Peak Floor Velocity", - } - - # conversion dictionary for demand unit names - convert_demand_unit = { - 'Unit less': 'unitless', - 'Radians': 'rad', - 'g': 'g', - 'meter/sec': 'mps', - } - - # for each component... - # (this approach is not efficient, but easy to follow which was considered - # more important than efficiency.) - for cmp in df_db_source.itertuples(): - # create a dotted component index - ID = cmp.Index.split('.') - cmpID = f'{ID[0][0]}.{ID[0][1:3]}.{ID[0][3:5]}.{ID[1]}' - - # store the new index - df_db.loc[cmp.Index, 'Index'] = cmpID - - # assume the component information is complete - incomplete = False - - # store demand specifications - df_db.loc[cmp.Index, 'Demand-Type'] = convert_demand_type[ - cmp.Demand_Parameter_value - ] - df_db.loc[cmp.Index, 'Demand-Unit'] = convert_demand_unit[ - cmp.Demand_Parameter_unit - ] - df_db.loc[cmp.Index, 'Demand-Offset'] = int( - cmp.Demand_Location_use_floor_above_YesNo - ) - df_db.loc[cmp.Index, 'Demand-Directional'] = int(cmp.Directional) - - # parse the damage state hierarchy - DS_setup = parse_DS_Hierarchy(cmp.DS_Hierarchy) - - # get the raw metadata for the component - cmp_meta = df_meta.loc[cmp.Index, :] - - # store the global (i.e., not DS-specific) metadata - - # start with a comp. description - if not pd.isna(cmp_meta['Component_Description']): - comments = cmp_meta['Component_Description'] - else: - comments = '' - - # the additional fields are added to the description if they exist - - if cmp_meta['Construction_Quality'] != 'Not Specified': - comments += f'\nConstruction Quality: {cmp_meta["Construction_Quality"]}' - - if cmp_meta['Seismic_Installation_Conditions'] not in [ - 'Not Specified', - 'Not applicable', - 'Unknown', - 'Any', - ]: - comments += ( - f'\nSeismic Installation Conditions: ' - f'{cmp_meta["Seismic_Installation_Conditions"]}' - ) - - if cmp_meta['Comments__Notes'] != 'None': - comments += f'\nNotes: {cmp_meta["Comments__Notes"]}' - - if cmp_meta['Author'] not in ['Not Given', 'By User']: - comments += f'\nAuthor: {cmp_meta["Author"]}' - - # get the suggested block size and replace the misleading values with ea - block_size = cmp_meta['Fragility_Unit_of_Measure'].split(' ')[::-1] - - meta_data = { - "Description": cmp_meta['Component_Name'], - "Comments": comments, - "SuggestedComponentBlockSize": ' '.join(block_size), - "RoundUpToIntegerQuantity": cmp_meta['Round_to_Integer_Unit'], - "LimitStates": {}, - } - - # now look at each Limit State - for LS_i, LS_contents in enumerate(DS_setup): - LS_i = LS_i + 1 - LS_contents = np.atleast_1d(LS_contents) - - ls_meta = {} - - # start with the special cases with multiple DSs in an LS - if LS_contents[0] in {'MutEx', 'Simul'}: - # collect the fragility data for the member DSs - median_demands = [] - dispersions = [] - weights = [] - for ds in LS_contents[1:]: - median_demands.append(getattr(cmp, f"DS_{ds[2]}_Median_Demand")) - - dispersions.append( - getattr(cmp, f"DS_{ds[2]}_Total_Dispersion_Beta") - ) - - weights.append(getattr(cmp, f"DS_{ds[2]}_Probability")) - - # make sure the specified distribution parameters are appropriate - if (np.unique(median_demands).size != 1) or ( - np.unique(dispersions).size != 1 - ): - raise ValueError( - f"Incorrect mutually exclusive DS " - f"definition in component {cmp.Index} at " - f"Limit State {LS_i}" - ) - - if LS_contents[0] == 'MutEx': - # in mutually exclusive cases, make sure the specified DS - # weights sum up to one - np.testing.assert_allclose( - np.sum(np.array(weights, dtype=float)), - 1.0, - err_msg=f"Mutually exclusive Damage State weights do " - f"not sum to 1.0 in component {cmp.Index} at " - f"Limit State {LS_i}", - ) - - # and save all DS metadata under this Limit State - for ds in LS_contents[1:]: - ds_id = ds[2] - - repair_action = cmp_meta[f"DS_{ds_id}_Repair_Description"] - if pd.isna(repair_action): - repair_action = "" - - ls_meta.update( - { - f"DS{ds_id}": { - "Description": cmp_meta[ - f"DS_{ds_id}_Description" - ], - "RepairAction": repair_action, - } - } - ) - - else: - # in simultaneous cases, convert simultaneous weights into - # mutexc weights - sim_ds_count = len(LS_contents) - 1 - ds_count = 2 ** (sim_ds_count) - 1 - - sim_weights = [] - - for ds_id in range(1, ds_count + 1): - ds_map = format(ds_id, f'0{sim_ds_count}b') - - sim_weights.append( - np.product( - [ - ( - weights[ds_i] - if ds_map[-ds_i - 1] == '1' - else 1.0 - weights[ds_i] - ) - for ds_i in range(sim_ds_count) - ] - ) - ) - - # save ds metadata - we need to be clever here - # the original metadata is saved for the pure cases - # when only one DS is triggered - # all other DSs store information about which - # combination of pure DSs they represent - - if ds_map.count('1') == 1: - ds_pure_id = ds_map[::-1].find('1') + 1 - - repair_action = cmp_meta[ - f"DS_{ds_pure_id}_Repair_Description" - ] - if pd.isna(repair_action): - repair_action = "" - - ls_meta.update( - { - f"DS{ds_id}": { - "Description": f"Pure DS{ds_pure_id}. " - + cmp_meta[f"DS_{ds_pure_id}_Description"], - "RepairAction": repair_action, - } - } - ) - - else: - ds_combo = [ - f'DS{_.start() + 1}' - for _ in re.finditer('1', ds_map[::-1]) - ] - - ls_meta.update( - { - f"DS{ds_id}": { - "Description": 'Combination of ' - + ' & '.join(ds_combo), - "RepairAction": ( - 'Combination of pure DS repair actions.' - ), - } - } - ) - - # adjust weights to respect the assumption that at least - # one DS will occur (i.e., the case with all DSs returning - # False is not part of the event space) - sim_weights_array = np.array(sim_weights) / np.sum(sim_weights) - - weights = sim_weights_array - - theta_0 = median_demands[0] - theta_1 = dispersions[0] - weights_str = ' | '.join([f"{w:.6f}" for w in weights]) - - df_db.loc[cmp.Index, f'LS{LS_i}-DamageStateWeights'] = weights_str - - # then look at the sequential DS cases - elif LS_contents[0].startswith('DS'): - # this is straightforward, store the data in the table and dict - ds_id = LS_contents[0][2] - - theta_0 = getattr(cmp, f"DS_{ds_id}_Median_Demand") - theta_1 = getattr(cmp, f"DS_{ds_id}_Total_Dispersion_Beta") - - repair_action = cmp_meta[f"DS_{ds_id}_Repair_Description"] - if pd.isna(repair_action): - repair_action = "" - - ls_meta.update( - { - f"DS{ds_id}": { - "Description": cmp_meta[f"DS_{ds_id}_Description"], - "RepairAction": repair_action, - } - } - ) - - # FEMA P58 assumes lognormal distribution for every fragility - df_db.loc[cmp.Index, f'LS{LS_i}-Family'] = 'lognormal' - - # identify incomplete cases... - - # where theta is missing - if theta_0 != 'By User': - df_db.loc[cmp.Index, f'LS{LS_i}-Theta_0'] = theta_0 - else: - incomplete = True - - # where beta is missing - if theta_1 != 'By User': - df_db.loc[cmp.Index, f'LS{LS_i}-Theta_1'] = theta_1 - else: - incomplete = True - - # store the collected metadata for this limit state - meta_data['LimitStates'].update({f"LS{LS_i}": ls_meta}) - - # store the incomplete flag for this component - df_db.loc[cmp.Index, 'Incomplete'] = int(incomplete) - - # store the metadata for this component - meta_dict.update({cmpID: meta_data}) - - # assign the Index column as the new ID - df_db.set_index('Index', inplace=True) - - # rename the index - df_db.index.name = "ID" - - # convert to optimal datatypes to reduce file size - df_db = df_db.convert_dtypes() - - # save the fragility data - df_db.to_csv(target_data_file) - - # save the metadata - with open(target_meta_file, 'w+', encoding='utf-8') as f: - json.dump(meta_dict, f, indent=2) - - print("Successfully parsed and saved the fragility data from FEMA P58") - - -def create_FEMA_P58_repair_db( - source_file, - meta_file='', - target_data_file='loss_repair_DB_FEMA_P58_2nd.csv', - target_meta_file='loss_repair_DB_FEMA_P58_2nd.json', -): - """ - Create a repair consequence parameter database based on the FEMA P58 data - - The method was developed to process v3.1.2 of the FragilityDatabase xls - that is provided with FEMA P58 2nd edition. - - Parameters - ---------- - source_file: string - Path to the fragility database file. - meta_file: string - Path to the JSON file with metadata about the database. - target_data_file: string - Path where the consequence data file should be saved. A csv file is - expected. - target_meta_file: string - Path where the consequence metadata should be saved. A json file is - expected. - - """ - - # parse the source file - df = pd.concat( - [ - pd.read_excel(source_file, sheet_name=sheet, header=2, index_col=1) - for sheet in ('Summary', 'Cost Summary', 'Env Summary') - ], - axis=1, - ) - - # parse the extra metadata file - if Path(meta_file).is_file(): - with open(meta_file, 'r', encoding='utf-8') as f: - frag_meta = json.load(f) - else: - frag_meta = {} - - # remove duplicate columns - # (there are such because we joined two tables that were read separately) - df = df.loc[:, ~df.columns.duplicated()] - - # remove empty rows and columns - df.dropna(axis=0, how='all', inplace=True) - df.dropna(axis=1, how='all', inplace=True) - - # filter the columns we need for the repair database - cols_to_db = [ - "Fragility Unit of Measure", - 'DS Hierarchy', - ] - for DS_i in range(1, 6): - cols_to_db += [ - f"Best Fit, DS{DS_i}", - f"Lower Qty Mean, DS{DS_i}", - f"Upper Qty Mean, DS{DS_i}", - f"Lower Qty Cutoff, DS{DS_i}", - f"Upper Qty Cutoff, DS{DS_i}", - f"CV / Dispersion, DS{DS_i}", - # -------------------------- - f"Best Fit, DS{DS_i}.1", - f"Lower Qty Mean, DS{DS_i}.1", - f"Upper Qty Mean, DS{DS_i}.1", - f"Lower Qty Cutoff, DS{DS_i}.1", - f"Upper Qty Cutoff, DS{DS_i}.1", - f"CV / Dispersion, DS{DS_i}.2", - f"DS {DS_i}, Long Lead Time", - # -------------------------- - f'Repair Cost, p10, DS{DS_i}', - f'Repair Cost, p50, DS{DS_i}', - f'Repair Cost, p90, DS{DS_i}', - f'Time, p10, DS{DS_i}', - f'Time, p50, DS{DS_i}', - f'Time, p90, DS{DS_i}', - f'Mean Value, DS{DS_i}', - f'Mean Value, DS{DS_i}.1', - # -------------------------- - # Columns added for the Environmental loss - f"DS{DS_i} Best Fit", - f"DS{DS_i} CV or Beta", - # -------------------------- - f"DS{DS_i} Best Fit.1", - f"DS{DS_i} CV or Beta.1", - # -------------------------- - f"DS{DS_i} Embodied Carbon (kg CO2eq)", - f"DS{DS_i} Embodied Energy (MJ)", - ] - - # filter the columns that we need for the metadata - cols_to_meta = [ - "Component Name", - "Component Description", - "Construction Quality:", - "Seismic Installation Conditions:", - "Comments / Notes", - "Author", - "Fragility Unit of Measure", - "Round to Integer Unit?", - "DS 1, Description", - "DS 1, Repair Description", - "DS 2, Description", - "DS 2, Repair Description", - "DS 3, Description", - "DS 3, Repair Description", - "DS 4, Description", - "DS 4, Repair Description", - "DS 5, Description", - "DS 5, Repair Description", - ] - - # remove special characters to make it easier to work with column names - str_map = { - ord(' '): "_", - ord('.'): "_", - ord(':'): None, - ord('('): None, - ord(')'): None, - ord('?'): None, - ord('/'): None, - ord(','): None, - } - - df_db_source = df.loc[:, cols_to_db] - df_db_source.columns = [s.translate(str_map) for s in cols_to_db] - df_db_source.sort_index(inplace=True) - - df_meta = df.loc[:, cols_to_meta] - df_meta.columns = [s.translate(str_map) for s in cols_to_meta] - - df_db_source.replace('BY USER', np.nan, inplace=True) - - # initialize the output loss table - # define the columns - out_cols = [ - "Index", - "Incomplete", - "Quantity-Unit", - "DV-Unit", - ] - for DS_i in range(1, 16): - out_cols += [ - f"DS{DS_i}-Family", - f"DS{DS_i}-Theta_0", - f"DS{DS_i}-Theta_1", - f"DS{DS_i}-LongLeadTime", - ] - - # create the MultiIndex - comps = df_db_source.index.values - DVs = ['Cost', 'Time', 'Carbon', 'Energy'] - df_MI = pd.MultiIndex.from_product([comps, DVs], names=['ID', 'DV']) - - df_db = pd.DataFrame(columns=out_cols, index=df_MI, dtype=float) - - # initialize the dictionary that stores the loss metadata - meta_dict = {} - - # add the general information to the meta dict - if "_GeneralInformation" in frag_meta.keys(): - frag_meta = frag_meta["_GeneralInformation"] - - meta_dict.update({"_GeneralInformation": frag_meta}) - - convert_family = {'LogNormal': 'lognormal', 'Normal': 'normal'} - - # for each component... - # (this approach is not efficient, but easy to follow which was considered - # more important than efficiency.) - for cmp in df_db_source.itertuples(): - ID = cmp.Index.split('.') - cmpID = f'{ID[0][0]}.{ID[0][1:3]}.{ID[0][3:5]}.{ID[1]}' - - # store the new index - df_db.loc[cmp.Index, 'Index'] = cmpID - - # assume the component information is complete - incomplete_cost = False - incomplete_time = False - incomplete_carbon = False - incomplete_energy = False - - # store units - - df_db.loc[cmp.Index, 'Quantity-Unit'] = ' '.join( - cmp.Fragility_Unit_of_Measure.split(' ')[::-1] - ).strip() - df_db.loc[(cmp.Index, 'Cost'), 'DV-Unit'] = "USD_2011" - df_db.loc[(cmp.Index, 'Time'), 'DV-Unit'] = "worker_day" - df_db.loc[(cmp.Index, 'Carbon'), 'DV-Unit'] = "kg" - df_db.loc[(cmp.Index, 'Energy'), 'DV-Unit'] = "MJ" - - # get the raw metadata for the component - cmp_meta = df_meta.loc[cmp.Index, :] - - # store the global (i.e., not DS-specific) metadata - - # start with a comp. description - if not pd.isna(cmp_meta['Component_Description']): - comments = cmp_meta['Component_Description'] - else: - comments = '' - - # the additional fields are added to the description if they exist - if cmp_meta['Construction_Quality'] != 'Not Specified': - comments += ( - f'\nConstruction Quality: ' f'{cmp_meta["Construction_Quality"]}' - ) - - if cmp_meta['Seismic_Installation_Conditions'] not in [ - 'Not Specified', - 'Not applicable', - 'Unknown', - 'Any', - ]: - comments += ( - f'\nSeismic Installation Conditions: ' - f'{cmp_meta["Seismic_Installation_Conditions"]}' - ) - - if cmp_meta['Comments__Notes'] != 'None': - comments += f'\nNotes: {cmp_meta["Comments__Notes"]}' - - if cmp_meta['Author'] not in ['Not Given', 'By User']: - comments += f'\nAuthor: {cmp_meta["Author"]}' - - # get the suggested block size and replace the misleading values with ea - block_size = cmp_meta['Fragility_Unit_of_Measure'].split(' ')[::-1] - - meta_data = { - "Description": cmp_meta['Component_Name'], - "Comments": comments, - "SuggestedComponentBlockSize": ' '.join(block_size), - "RoundUpToIntegerQuantity": cmp_meta['Round_to_Integer_Unit'], - "ControllingDemand": "Damage Quantity", - "DamageStates": {}, - } - - # Handle components with simultaneous damage states separately - if 'Simul' in cmp.DS_Hierarchy: - # Note that we are assuming that all damage states are triggered by - # a single limit state in these components. - # This assumption holds for the second edition of FEMA P58, but it - # might need to be revisited in future editions. - - cost_est = {} - time_est = {} - carbon_est = {} - energy_est = {} - - # get the p10, p50, and p90 estimates for all damage states - for DS_i in range(1, 6): - if not pd.isna(getattr(cmp, f'Repair_Cost_p10_DS{DS_i}')): - cost_est.update( - { - f'DS{DS_i}': np.array( - [ - getattr(cmp, f'Repair_Cost_p10_DS{DS_i}'), - getattr(cmp, f'Repair_Cost_p50_DS{DS_i}'), - getattr(cmp, f'Repair_Cost_p90_DS{DS_i}'), - getattr(cmp, f'Lower_Qty_Mean_DS{DS_i}'), - getattr(cmp, f'Upper_Qty_Mean_DS{DS_i}'), - ] - ) - } - ) - - time_est.update( - { - f'DS{DS_i}': np.array( - [ - getattr(cmp, f'Time_p10_DS{DS_i}'), - getattr(cmp, f'Time_p50_DS{DS_i}'), - getattr(cmp, f'Time_p90_DS{DS_i}'), - getattr(cmp, f'Lower_Qty_Mean_DS{DS_i}_1'), - getattr(cmp, f'Upper_Qty_Mean_DS{DS_i}_1'), - int( - getattr(cmp, f'DS_{DS_i}_Long_Lead_Time') - == 'YES' - ), - ] - ) - } - ) - - if not pd.isna(getattr(cmp, f'DS{DS_i}_Embodied_Carbon_kg_CO2eq')): - theta_0, theta_1, family = [ - getattr(cmp, f'DS{DS_i}_Embodied_Carbon_kg_CO2eq'), - getattr(cmp, f'DS{DS_i}_CV_or_Beta'), - getattr(cmp, f'DS{DS_i}_Best_Fit'), - ] - - if family == 'Normal': - p10, p50, p90 = norm.ppf( - [0.1, 0.5, 0.9], loc=theta_0, scale=theta_0 * theta_1 - ) - elif family == 'LogNormal': - p10, p50, p90 = np.exp( - norm.ppf( - [0.1, 0.5, 0.9], loc=np.log(theta_0), scale=theta_1 - ) - ) - - carbon_est.update({f'DS{DS_i}': np.array([p10, p50, p90])}) - - if not pd.isna(getattr(cmp, f'DS{DS_i}_Embodied_Energy_MJ')): - theta_0, theta_1, family = [ - getattr(cmp, f'DS{DS_i}_Embodied_Energy_MJ'), - getattr(cmp, f'DS{DS_i}_CV_or_Beta_1'), - getattr(cmp, f'DS{DS_i}_Best_Fit_1'), - ] - - if family == 'Normal': - p10, p50, p90 = norm.ppf( - [0.1, 0.5, 0.9], loc=theta_0, scale=theta_0 * theta_1 - ) - elif family == 'LogNormal': - p10, p50, p90 = np.exp( - norm.ppf( - [0.1, 0.5, 0.9], loc=np.log(theta_0), scale=theta_1 - ) - ) - - energy_est.update({f'DS{DS_i}': np.array([p10, p50, p90])}) - - # now prepare the equivalent mutex damage states - sim_ds_count = len(cost_est.keys()) - ds_count = 2 ** (sim_ds_count) - 1 - - for DS_i in range(1, ds_count + 1): - ds_map = format(DS_i, f'0{sim_ds_count}b') - - cost_vals = np.sum( - [ - ( - cost_est[f'DS{ds_i + 1}'] - if ds_map[-ds_i - 1] == '1' - else np.zeros(5) - ) - for ds_i in range(sim_ds_count) - ], - axis=0, - ) - - time_vals = np.sum( - [ - ( - time_est[f'DS{ds_i + 1}'] - if ds_map[-ds_i - 1] == '1' - else np.zeros(6) - ) - for ds_i in range(sim_ds_count) - ], - axis=0, - ) - - carbon_vals = np.sum( - [ - ( - carbon_est[f'DS{ds_i + 1}'] - if ds_map[-ds_i - 1] == '1' - else np.zeros(3) - ) - for ds_i in range(sim_ds_count) - ], - axis=0, - ) - - energy_vals = np.sum( - [ - ( - energy_est[f'DS{ds_i + 1}'] - if ds_map[-ds_i - 1] == '1' - else np.zeros(3) - ) - for ds_i in range(sim_ds_count) - ], - axis=0, - ) - - # fit a distribution - family_hat, theta_hat = fit_distribution_to_percentiles( - cost_vals[:3], [0.1, 0.5, 0.9], ['normal', 'lognormal'] - ) - - cost_theta = theta_hat - if family_hat == 'normal': - cost_theta[1] = cost_theta[1] / cost_theta[0] - - time_theta = [ - time_vals[1], - np.sqrt(cost_theta[1] ** 2.0 + 0.25**2.0), - ] - - # fit distributions to environmental impact consequences - ( - family_hat_carbon, - theta_hat_carbon, - ) = fit_distribution_to_percentiles( - carbon_vals[:3], [0.1, 0.5, 0.9], ['normal', 'lognormal'] - ) - - carbon_theta = theta_hat_carbon - if family_hat_carbon == 'normal': - carbon_theta[1] = carbon_theta[1] / carbon_theta[0] - - ( - family_hat_energy, - theta_hat_energy, - ) = fit_distribution_to_percentiles( - energy_vals[:3], [0.1, 0.5, 0.9], ['normal', 'lognormal'] - ) - - energy_theta = theta_hat_energy - if family_hat_energy == 'normal': - energy_theta[1] = energy_theta[1] / energy_theta[0] - - # Note that here we assume that the cutoff quantities are - # identical across damage states. - # This assumption holds for the second edition of FEMA P58, but - # it might need to be revisited in future editions. - cost_qnt_low = getattr(cmp, 'Lower_Qty_Cutoff_DS1') - cost_qnt_up = getattr(cmp, 'Upper_Qty_Cutoff_DS1') - time_qnt_low = getattr(cmp, 'Lower_Qty_Cutoff_DS1_1') - time_qnt_up = getattr(cmp, 'Upper_Qty_Cutoff_DS1_1') - - # store the results - df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Family'] = family_hat - - df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Theta_0'] = ( - f"{cost_vals[3]:g},{cost_vals[4]:g}|" - f"{cost_qnt_low:g},{cost_qnt_up:g}" - ) - - df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Theta_1'] = ( - f"{cost_theta[1]:g}" - ) - - df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Family'] = family_hat - - df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Theta_0'] = ( - f"{time_vals[3]:g},{time_vals[4]:g}|" - f"{time_qnt_low:g},{time_qnt_up:g}" - ) - - df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Theta_1'] = ( - f"{time_theta[1]:g}" - ) - - df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-LongLeadTime'] = int( - time_vals[5] > 0 - ) - - df_db.loc[(cmp.Index, 'Carbon'), f'DS{DS_i}-Family'] = ( - family_hat_carbon - ) - - df_db.loc[(cmp.Index, 'Carbon'), f'DS{DS_i}-Theta_0'] = ( - f"{carbon_theta[0]:g}" - ) - - df_db.loc[(cmp.Index, 'Carbon'), f'DS{DS_i}-Theta_1'] = ( - f"{carbon_theta[1]:g}" - ) - - df_db.loc[(cmp.Index, 'Energy'), f'DS{DS_i}-Family'] = ( - family_hat_energy - ) - - df_db.loc[(cmp.Index, 'Energy'), f'DS{DS_i}-Theta_0'] = ( - f"{energy_theta[0]:g}" - ) - - df_db.loc[(cmp.Index, 'Energy'), f'DS{DS_i}-Theta_1'] = ( - f"{energy_theta[1]:g}" - ) - - if ds_map.count('1') == 1: - ds_pure_id = ds_map[::-1].find('1') + 1 - - repair_action = cmp_meta[f"DS_{ds_pure_id}_Repair_Description"] - if pd.isna(repair_action): - repair_action = "" - - meta_data['DamageStates'].update( - { - f"DS{DS_i}": { - "Description": f"Pure DS{ds_pure_id}. " - + cmp_meta[f"DS_{ds_pure_id}_Description"], - "RepairAction": repair_action, - } - } - ) - - else: - ds_combo = [ - f'DS{_.start() + 1}' for _ in re.finditer('1', ds_map[::-1]) - ] - - meta_data['DamageStates'].update( - { - f"DS{DS_i}": { - "Description": 'Combination of ' - + ' & '.join(ds_combo), - "RepairAction": 'Combination of pure DS repair ' - 'actions.', - } - } - ) - - # for every other component... - else: - # now look at each Damage State - for DS_i in range(1, 6): - # cost - if not pd.isna(getattr(cmp, f'Best_Fit_DS{DS_i}')): - df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Family'] = ( - convert_family[getattr(cmp, f'Best_Fit_DS{DS_i}')] - ) - - if not pd.isna(getattr(cmp, f'Lower_Qty_Mean_DS{DS_i}')): - theta_0_low = getattr(cmp, f'Lower_Qty_Mean_DS{DS_i}') - theta_0_up = getattr(cmp, f'Upper_Qty_Mean_DS{DS_i}') - qnt_low = getattr(cmp, f'Lower_Qty_Cutoff_DS{DS_i}') - qnt_up = getattr(cmp, f'Upper_Qty_Cutoff_DS{DS_i}') - - if theta_0_low == 0.0 and theta_0_up == 0.0: - df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Family'] = ( - np.nan - ) - - else: - df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Theta_0'] = ( - f"{theta_0_low:g},{theta_0_up:g}|" - f"{qnt_low:g},{qnt_up:g}" - ) - - df_db.loc[(cmp.Index, 'Cost'), f'DS{DS_i}-Theta_1'] = ( - f"{getattr(cmp, f'CV__Dispersion_DS{DS_i}'):g}" - ) - - else: - incomplete_cost = True - - repair_action = cmp_meta[f"DS_{DS_i}_Repair_Description"] - if pd.isna(repair_action): - repair_action = "" - - meta_data['DamageStates'].update( - { - f"DS{DS_i}": { - "Description": cmp_meta[f"DS_{DS_i}_Description"], - "RepairAction": repair_action, - } - } - ) - - # time - if not pd.isna(getattr(cmp, f'Best_Fit_DS{DS_i}_1')): - df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Family'] = ( - convert_family[getattr(cmp, f'Best_Fit_DS{DS_i}_1')] - ) - - if not pd.isna(getattr(cmp, f'Lower_Qty_Mean_DS{DS_i}_1')): - theta_0_low = getattr(cmp, f'Lower_Qty_Mean_DS{DS_i}_1') - theta_0_up = getattr(cmp, f'Upper_Qty_Mean_DS{DS_i}_1') - qnt_low = getattr(cmp, f'Lower_Qty_Cutoff_DS{DS_i}_1') - qnt_up = getattr(cmp, f'Upper_Qty_Cutoff_DS{DS_i}_1') - - if theta_0_low == 0.0 and theta_0_up == 0.0: - df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Family'] = ( - np.nan - ) - - else: - df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Theta_0'] = ( - f"{theta_0_low:g},{theta_0_up:g}|" - f"{qnt_low:g},{qnt_up:g}" - ) - - df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-Theta_1'] = ( - f"{getattr(cmp, f'CV__Dispersion_DS{DS_i}_2'):g}" - ) - - df_db.loc[(cmp.Index, 'Time'), f'DS{DS_i}-LongLeadTime'] = ( - int(getattr(cmp, f'DS_{DS_i}_Long_Lead_Time') == 'YES') - ) - - else: - incomplete_time = True - - # Carbon - if not pd.isna(getattr(cmp, f'DS{DS_i}_Best_Fit')): - df_db.loc[(cmp.Index, 'Carbon'), f'DS{DS_i}-Family'] = ( - convert_family[getattr(cmp, f'DS{DS_i}_Best_Fit')] - ) - - df_db.loc[(cmp.Index, 'Carbon'), f'DS{DS_i}-Theta_0'] = getattr( - cmp, f'DS{DS_i}_Embodied_Carbon_kg_CO2eq' - ) - - df_db.loc[(cmp.Index, 'Carbon'), f'DS{DS_i}-Theta_1'] = getattr( - cmp, f'DS{DS_i}_CV_or_Beta' - ) - - # Energy - if not pd.isna(getattr(cmp, f'DS{DS_i}_Best_Fit_1')): - df_db.loc[(cmp.Index, 'Energy'), f'DS{DS_i}-Family'] = ( - convert_family[getattr(cmp, f'DS{DS_i}_Best_Fit_1')] - ) - - df_db.loc[(cmp.Index, 'Energy'), f'DS{DS_i}-Theta_0'] = getattr( - cmp, f'DS{DS_i}_Embodied_Energy_MJ' - ) - - df_db.loc[(cmp.Index, 'Energy'), f'DS{DS_i}-Theta_1'] = getattr( - cmp, f'DS{DS_i}_CV_or_Beta_1' - ) - - df_db.loc[(cmp.Index, 'Cost'), 'Incomplete'] = int(incomplete_cost) - df_db.loc[(cmp.Index, 'Time'), 'Incomplete'] = int(incomplete_time) - df_db.loc[(cmp.Index, 'Carbon'), 'Incomplete'] = int(incomplete_carbon) - df_db.loc[(cmp.Index, 'Energy'), 'Incomplete'] = int(incomplete_energy) - # store the metadata for this component - meta_dict.update({cmpID: meta_data}) - - # assign the Index column as the new ID - df_db.index = pd.MultiIndex.from_arrays( - [df_db['Index'].values, df_db.index.get_level_values(1)] - ) - - df_db.drop('Index', axis=1, inplace=True) - - # review the database and drop rows with no information - cmp_to_drop = [] - for cmp in df_db.index: - empty = True - - for DS_i in range(1, 6): - if not pd.isna(df_db.loc[cmp, f'DS{DS_i}-Family']): - empty = False - break - - if empty: - cmp_to_drop.append(cmp) - - df_db.drop(cmp_to_drop, axis=0, inplace=True) - for cmp in cmp_to_drop: - if cmp[0] in meta_dict: - del meta_dict[cmp[0]] - - # convert to optimal datatypes to reduce file size - df_db = df_db.convert_dtypes() - - df_db = base.convert_to_SimpleIndex(df_db, 0) - - # rename the index - df_db.index.name = "ID" - - # save the consequence data - df_db.to_csv(target_data_file) - - # save the metadata - with open(target_meta_file, 'w+', encoding='utf-8') as f: - json.dump(meta_dict, f, indent=2) - - print("Successfully parsed and saved the repair consequence data from FEMA P58") - - -def create_Hazus_EQ_fragility_db( - source_file, - meta_file='', - target_data_file='damage_DB_Hazus_EQ_bldg.csv', - target_meta_file='damage_DB_Hazus_EQ_bldg.json', - resolution='building', -): - """ - Create a database file based on the HAZUS EQ Technical Manual - - This method was developed to process a json file with tabulated data from - v5.1 of the Hazus Earthquake Technical Manual. The json file is included - under data_sources in the SimCenter DB_DamageAndLoss repo on GitHub. - - Parameters - ---------- - source_file: string - Path to the fragility database file. - meta_file: string - Path to the JSON file with metadata about the database. - target_data_file: string - Path where the fragility data file should be saved. A csv file is - expected. - target_meta_file: string - Path where the fragility metadata should be saved. A json file is - expected. - resolution: string - If building, the function produces the conventional Hazus - fragilities. If story, the function produces story-level - fragilities. - - """ - - # adjust the target filenames if needed - if resolution == 'story': - target_data_file = target_data_file.replace('bldg', 'story') - target_meta_file = target_meta_file.replace('bldg', 'story') - - # parse the source file - with open(source_file, 'r', encoding='utf-8') as f: - raw_data = json.load(f) - - # parse the extra metadata file - if Path(meta_file).is_file(): - with open(meta_file, 'r', encoding='utf-8') as f: - frag_meta = json.load(f) - else: - frag_meta = {} - - # prepare lists of labels for various building features - design_levels = list( - raw_data['Structural_Fragility_Groups']['EDP_limits'].keys() - ) - - building_types = list( - raw_data['Structural_Fragility_Groups']['P_collapse'].keys() - ) - - convert_design_level = { - 'High_code': 'HC', - 'Moderate_code': 'MC', - 'Low_code': 'LC', - 'Pre_code': 'PC', - } - - # initialize the fragility table - df_db = pd.DataFrame( - columns=[ - "ID", - "Incomplete", - "Demand-Type", - "Demand-Unit", - "Demand-Offset", - "Demand-Directional", - "LS1-Family", - "LS1-Theta_0", - "LS1-Theta_1", - "LS1-DamageStateWeights", - "LS2-Family", - "LS2-Theta_0", - "LS2-Theta_1", - "LS2-DamageStateWeights", - "LS3-Family", - "LS3-Theta_0", - "LS3-Theta_1", - "LS3-DamageStateWeights", - "LS4-Family", - "LS4-Theta_0", - "LS4-Theta_1", - "LS4-DamageStateWeights", - ], - index=np.arange(len(building_types) * len(design_levels) * 5), - dtype=float, - ) - - # initialize the dictionary that stores the fragility metadata - meta_dict = {} - - # add the general information to the meta dict - if "_GeneralInformation" in frag_meta.keys(): - GI = frag_meta["_GeneralInformation"] - - # remove the decision variable part from the general info - GI.pop("DecisionVariables", None) - - for key, item in deepcopy(GI).items(): - if key == 'ComponentGroups_Damage': - GI.update({'ComponentGroups': item}) - - if key.startswith('ComponentGroups'): - GI.pop(key, None) - - meta_dict.update({"_GeneralInformation": GI}) - - counter = 0 - - # First, prepare the structural fragilities - S_data = raw_data['Structural_Fragility_Groups'] - - for bt in building_types: - for dl in design_levels: - if bt in S_data['EDP_limits'][dl].keys(): - # add a dot in bt between structure and height labels, if needed - if (len(bt) > 2) and (bt[-1] in {'L', 'M', 'H'}): - bt_exp = f'{bt[:-1]}.{bt[-1]}' - st = bt[:-1] - hc = bt[-1] - else: - bt_exp = bt - st = bt - hc = None - - # story-level fragilities are based only on the low rise archetypes - if resolution == 'story': - if hc in {'M', 'H'}: - continue - if hc == 'L': - bt_exp = st - - # create the component id - cmp_id = f'STR.{bt_exp}.{convert_design_level[dl]}' - df_db.loc[counter, 'ID'] = cmp_id - - # store demand specifications - if resolution == 'building': - df_db.loc[counter, 'Demand-Type'] = "Peak Roof Drift Ratio" - elif resolution == 'story': - df_db.loc[counter, 'Demand-Type'] = "Peak Interstory Drift Ratio" - - df_db.loc[counter, 'Demand-Unit'] = "rad" - df_db.loc[counter, 'Demand-Offset'] = 0 - - # add metadata - if hc is not None: - cmp_meta = { - "Description": ( - frag_meta['Meta']['Collections']['STR']['Description'] - + ", " - + frag_meta['Meta']['StructuralSystems'][st][ - 'Description' - ] - + ", " - + frag_meta['Meta']['HeightClasses'][hc]['Description'] - + ", " - + frag_meta['Meta']['DesignLevels'][ - convert_design_level[dl] - ]['Description'] - ), - "Comments": ( - frag_meta['Meta']['Collections']['STR']['Comment'] - + "\n" - + frag_meta['Meta']['StructuralSystems'][st]['Comment'] - + "\n" - + frag_meta['Meta']['HeightClasses'][hc]['Comment'] - + "\n" - + frag_meta['Meta']['DesignLevels'][ - convert_design_level[dl] - ]['Comment'] - ), - "SuggestedComponentBlockSize": "1 EA", - "RoundUpToIntegerQuantity": "True", - "LimitStates": {}, - } - else: - cmp_meta = { - "Description": ( - frag_meta['Meta']['Collections']['STR']['Description'] - + ", " - + frag_meta['Meta']['StructuralSystems'][st][ - 'Description' - ] - + ", " - + frag_meta['Meta']['DesignLevels'][ - convert_design_level[dl] - ]['Description'] - ), - "Comments": ( - frag_meta['Meta']['Collections']['STR']['Comment'] - + "\n" - + frag_meta['Meta']['StructuralSystems'][st]['Comment'] - + "\n" - + frag_meta['Meta']['DesignLevels'][ - convert_design_level[dl] - ]['Comment'] - ), - "SuggestedComponentBlockSize": "1 EA", - "RoundUpToIntegerQuantity": "True", - "LimitStates": {}, - } - - # store the Limit State parameters - ds_meta = frag_meta['Meta']['StructuralSystems'][st]['DamageStates'] - for LS_i in range(1, 5): - df_db.loc[counter, f'LS{LS_i}-Family'] = 'lognormal' - df_db.loc[counter, f'LS{LS_i}-Theta_0'] = S_data['EDP_limits'][ - dl - ][bt][LS_i - 1] - df_db.loc[counter, f'LS{LS_i}-Theta_1'] = S_data[ - 'Fragility_beta' - ][dl] - - if LS_i == 4: - p_coll = S_data['P_collapse'][bt] - df_db.loc[counter, f'LS{LS_i}-DamageStateWeights'] = ( - f'{1.0 - p_coll} | {p_coll}' - ) - - cmp_meta["LimitStates"].update( - { - "LS4": { - "DS4": {"Description": ds_meta['DS4']}, - "DS5": {"Description": ds_meta['DS5']}, - } - } - ) - - else: - cmp_meta["LimitStates"].update( - { - f"LS{LS_i}": { - f"DS{LS_i}": { - "Description": ds_meta[f"DS{LS_i}"] - } - } - } - ) - - # store metadata - meta_dict.update({cmp_id: cmp_meta}) - - counter += 1 - - # Second, the non-structural drift sensitive one - NSD_data = raw_data['NonStructural_Drift_Sensitive_Fragility_Groups'] - - # create the component id - df_db.loc[counter, 'ID'] = 'NSD' - - # store demand specifications - if resolution == 'building': - df_db.loc[counter, 'Demand-Type'] = "Peak Roof Drift Ratio" - elif resolution == 'story': - df_db.loc[counter, 'Demand-Type'] = "Peak Interstory Drift Ratio" - - df_db.loc[counter, 'Demand-Unit'] = "rad" - df_db.loc[counter, 'Demand-Offset'] = 0 - - # add metadata - cmp_meta = { - "Description": frag_meta['Meta']['Collections']['NSD']['Description'], - "Comments": frag_meta['Meta']['Collections']['NSD']['Comment'], - "SuggestedComponentBlockSize": "1 EA", - "RoundUpToIntegerQuantity": "True", - "LimitStates": {}, - } - - # store the Limit State parameters - ds_meta = frag_meta['Meta']['Collections']['NSD']['DamageStates'] - for LS_i in range(1, 5): - df_db.loc[counter, f'LS{LS_i}-Family'] = 'lognormal' - df_db.loc[counter, f'LS{LS_i}-Theta_0'] = NSD_data['EDP_limits'][LS_i - 1] - df_db.loc[counter, f'LS{LS_i}-Theta_1'] = NSD_data['Fragility_beta'] - - # add limit state metadata - cmp_meta["LimitStates"].update( - {f"LS{LS_i}": {f"DS{LS_i}": {"Description": ds_meta[f"DS{LS_i}"]}}} - ) - - # store metadata - meta_dict.update({'NSD': cmp_meta}) - - counter += 1 - - # Third, the non-structural acceleration sensitive fragilities - NSA_data = raw_data['NonStructural_Acceleration_Sensitive_Fragility_Groups'] - - for dl in design_levels: - # create the component id - cmp_id = f'NSA.{convert_design_level[dl]}' - df_db.loc[counter, 'ID'] = cmp_id - - # store demand specifications - df_db.loc[counter, 'Demand-Type'] = "Peak Floor Acceleration" - df_db.loc[counter, 'Demand-Unit'] = "g" - df_db.loc[counter, 'Demand-Offset'] = 0 - - # add metadata - cmp_meta = { - "Description": ( - frag_meta['Meta']['Collections']['NSA']['Description'] - + ", " - + frag_meta['Meta']['DesignLevels'][convert_design_level[dl]][ - 'Description' - ] - ), - "Comments": ( - frag_meta['Meta']['Collections']['NSA']['Comment'] - + "\n" - + frag_meta['Meta']['DesignLevels'][convert_design_level[dl]][ - 'Comment' - ] - ), - "SuggestedComponentBlockSize": "1 EA", - "RoundUpToIntegerQuantity": "True", - "LimitStates": {}, - } - - # store the Limit State parameters - ds_meta = frag_meta['Meta']['Collections']['NSA']['DamageStates'] - for LS_i in range(1, 5): - df_db.loc[counter, f'LS{LS_i}-Family'] = 'lognormal' - df_db.loc[counter, f'LS{LS_i}-Theta_0'] = NSA_data['EDP_limits'][dl][ - LS_i - 1 - ] - df_db.loc[counter, f'LS{LS_i}-Theta_1'] = NSA_data['Fragility_beta'] - - # add limit state metadata - cmp_meta["LimitStates"].update( - {f"LS{LS_i}": {f"DS{LS_i}": {"Description": ds_meta[f"DS{LS_i}"]}}} - ) - - # store metadata - meta_dict.update({cmp_id: cmp_meta}) - - counter += 1 - - # Fourth, the lifeline facilities - only at the building-level resolution - if resolution == 'building': - LF_data = raw_data['Lifeline_Facilities'] - - for bt in building_types: - for dl in design_levels: - if bt in LF_data['EDP_limits'][dl].keys(): - # add a dot in bt between structure and height labels, if needed - if (len(bt) > 2) and (bt[-1] in {'L', 'M', 'H'}): - bt_exp = f'{bt[:-1]}.{bt[-1]}' - st = bt[:-1] - hc = bt[-1] - else: - bt_exp = bt - st = bt - hc = None - - # create the component id - cmp_id = f'LF.{bt_exp}.{convert_design_level[dl]}' - df_db.loc[counter, 'ID'] = cmp_id - - # store demand specifications - df_db.loc[counter, 'Demand-Type'] = "Peak Ground Acceleration" - df_db.loc[counter, 'Demand-Unit'] = "g" - df_db.loc[counter, 'Demand-Offset'] = 0 - - # add metadata - if hc is not None: - cmp_meta = { - "Description": ( - frag_meta['Meta']['Collections']['LF']['Description'] - + ", " - + frag_meta['Meta']['StructuralSystems'][st][ - 'Description' - ] - + ", " - + frag_meta['Meta']['HeightClasses'][hc][ - 'Description' - ] - + ", " - + frag_meta['Meta']['DesignLevels'][ - convert_design_level[dl] - ]['Description'] - ), - "Comments": ( - frag_meta['Meta']['Collections']['LF']['Comment'] - + "\n" - + frag_meta['Meta']['StructuralSystems'][st][ - 'Comment' - ] - + "\n" - + frag_meta['Meta']['HeightClasses'][hc]['Comment'] - + "\n" - + frag_meta['Meta']['DesignLevels'][ - convert_design_level[dl] - ]['Comment'] - ), - "SuggestedComponentBlockSize": "1 EA", - "RoundUpToIntegerQuantity": "True", - "LimitStates": {}, - } - else: - cmp_meta = { - "Description": ( - frag_meta['Meta']['Collections']['LF']['Description'] - + ", " - + frag_meta['Meta']['StructuralSystems'][st][ - 'Description' - ] - + ", " - + frag_meta['Meta']['DesignLevels'][ - convert_design_level[dl] - ]['Description'] - ), - "Comments": ( - frag_meta['Meta']['Collections']['LF']['Comment'] - + "\n" - + frag_meta['Meta']['StructuralSystems'][st][ - 'Comment' - ] - + "\n" - + frag_meta['Meta']['DesignLevels'][ - convert_design_level[dl] - ]['Comment'] - ), - "SuggestedComponentBlockSize": "1 EA", - "RoundUpToIntegerQuantity": "True", - "LimitStates": {}, - } - - # store the Limit State parameters - ds_meta = frag_meta['Meta']['StructuralSystems'][st][ - 'DamageStates' - ] - for LS_i in range(1, 5): - df_db.loc[counter, f'LS{LS_i}-Family'] = 'lognormal' - df_db.loc[counter, f'LS{LS_i}-Theta_0'] = LF_data[ - 'EDP_limits' - ][dl][bt][LS_i - 1] - df_db.loc[counter, f'LS{LS_i}-Theta_1'] = LF_data[ - 'Fragility_beta' - ][dl] - - if LS_i == 4: - p_coll = LF_data['P_collapse'][bt] - df_db.loc[counter, f'LS{LS_i}-DamageStateWeights'] = ( - f'{1.0 - p_coll} | {p_coll}' - ) - - cmp_meta["LimitStates"].update( - { - "LS4": { - "DS4": {"Description": ds_meta['DS4']}, - "DS5": {"Description": ds_meta['DS5']}, - } - } - ) - - else: - cmp_meta["LimitStates"].update( - { - f"LS{LS_i}": { - f"DS{LS_i}": { - "Description": ds_meta[f"DS{LS_i}"] - } - } - } - ) - - # store metadata - meta_dict.update({cmp_id: cmp_meta}) - - counter += 1 - - # Fifth, the ground failure fragilities - GF_data = raw_data['Ground_Failure'] - - for direction in ('Horizontal', 'Vertical'): - for f_depth in ('Shallow', 'Deep'): - # create the component id - cmp_id = f'GF.{direction[0]}.{f_depth[0]}' - df_db.loc[counter, 'ID'] = cmp_id - - # store demand specifications - df_db.loc[counter, 'Demand-Type'] = "Permanent Ground Deformation" - df_db.loc[counter, 'Demand-Unit'] = "inch" - df_db.loc[counter, 'Demand-Offset'] = 0 - - # add metadata - cmp_meta = { - "Description": ( - frag_meta['Meta']['Collections']['GF']['Description'] - + f", {direction} Direction, {f_depth} Foundation" - ), - "Comments": (frag_meta['Meta']['Collections']['GF']['Comment']), - "SuggestedComponentBlockSize": "1 EA", - "RoundUpToIntegerQuantity": "True", - "LimitStates": {}, - } - - # store the Limit State parameters - ds_meta = frag_meta['Meta']['Collections']['GF']['DamageStates'] - - df_db.loc[counter, 'LS1-Family'] = 'lognormal' - df_db.loc[counter, 'LS1-Theta_0'] = GF_data['EDP_limits'][direction][ - f_depth - ] - df_db.loc[counter, 'LS1-Theta_1'] = GF_data['Fragility_beta'][direction][ - f_depth - ] - p_complete = GF_data['P_Complete'] - df_db.loc[counter, 'LS1-DamageStateWeights'] = ( - f'{1.0 - p_complete} | {p_complete}' - ) - - cmp_meta["LimitStates"].update( - { - "LS1": { - "DS1": {"Description": ds_meta['DS1']}, - "DS2": {"Description": ds_meta['DS2']}, - } - } - ) - - # store metadata - meta_dict.update({cmp_id: cmp_meta}) - - counter += 1 - - # remove empty rows (from the end) - df_db.dropna(how='all', inplace=True) - - # All Hazus components have complete fragility info, - df_db['Incomplete'] = 0 - - # none of them are directional, - df_db['Demand-Directional'] = 0 - - # rename the index - df_db.set_index("ID", inplace=True) - - # convert to optimal datatypes to reduce file size - df_db = df_db.convert_dtypes() - - # save the fragility data - df_db.to_csv(target_data_file) - - # save the metadata - with open(target_meta_file, 'w+', encoding='utf-8') as f: - json.dump(meta_dict, f, indent=2) - - print("Successfully parsed and saved the fragility data from Hazus EQ") - - -def create_Hazus_EQ_repair_db( - source_file, - meta_file='', - target_data_file='loss_repair_DB_Hazus_EQ_bldg.csv', - target_meta_file='loss_repair_DB_Hazus_EQ_bldg.json', - resolution='building', -): - """ - Create a database file based on the HAZUS EQ Technical Manual - - This method was developed to process a json file with tabulated data from - v4.2.3 of the Hazus Earthquake Technical Manual. The json file is included - under data_sources in the SimCenter DB_DamageAndLoss repo on GitHub. - - Parameters - ---------- - source_file: string - Path to the Hazus database file. - meta_file: string - Path to the JSON file with metadata about the database. - target_data_file: string - Path where the repair DB file should be saved. A csv file is - expected. - target_meta_file: string - Path where the repair DB metadata should be saved. A json file is - expected. - resolution: string - If building, the function produces the conventional Hazus - fragilities. If story, the function produces story-level - fragilities. - - """ - - # adjust the target filenames if needed - if resolution == 'story': - target_data_file = target_data_file.replace('bldg', 'story') - target_meta_file = target_meta_file.replace('bldg', 'story') - - # parse the source file - with open(source_file, 'r', encoding='utf-8') as f: - raw_data = json.load(f) - - # parse the extra metadata file - if Path(meta_file).is_file(): - with open(meta_file, 'r', encoding='utf-8') as f: - frag_meta = json.load(f) - else: - frag_meta = {} - - # prepare lists of labels for various building features - occupancies = list(raw_data['Structural_Fragility_Groups']['Repair_cost'].keys()) - - # initialize the output loss table - # define the columns - out_cols = [ - "Incomplete", - "Quantity-Unit", - "DV-Unit", - ] - for DS_i in range(1, 6): - out_cols += [ - f"DS{DS_i}-Theta_0", - ] - - # create the MultiIndex - cmp_types = ['STR', 'NSD', 'NSA', 'LF'] - comps = [ - f'{cmp_type}.{occ_type}' - for cmp_type in cmp_types - for occ_type in occupancies - ] - DVs = ['Cost', 'Time'] - df_MI = pd.MultiIndex.from_product([comps, DVs], names=['ID', 'DV']) - - df_db = pd.DataFrame(columns=out_cols, index=df_MI, dtype=float) - - # initialize the dictionary that stores the loss metadata - meta_dict = {} - - # add the general information to the meta dict - if "_GeneralInformation" in frag_meta.keys(): - GI = frag_meta["_GeneralInformation"] - - for key, item in deepcopy(GI).items(): - if key == 'ComponentGroups_Loss_Repair': - GI.update({'ComponentGroups': item}) - - if key.startswith('ComponentGroups'): - GI.pop(key, None) - - meta_dict.update({"_GeneralInformation": GI}) - - # First, prepare the structural damage consequences - S_data = raw_data['Structural_Fragility_Groups'] - - for occ_type in occupancies: - # create the component id - cmp_id = f'STR.{occ_type}' - - cmp_meta = { - "Description": ( - frag_meta['Meta']['Collections']['STR']['Description'] - + ", " - + frag_meta['Meta']['OccupancyTypes'][occ_type]['Description'] - ), - "Comments": ( - frag_meta['Meta']['Collections']['STR']['Comment'] - + "\n" - + frag_meta['Meta']['OccupancyTypes'][occ_type]['Comment'] - ), - "SuggestedComponentBlockSize": "1 EA", - "RoundUpToIntegerQuantity": "True", - "DamageStates": {}, - } - - # store the consequence values for each Damage State - ds_meta = frag_meta['Meta']['Collections']['STR']['DamageStates'] - for DS_i in range(1, 6): - cmp_meta["DamageStates"].update( - {f"DS{DS_i}": {"Description": ds_meta[f"DS{DS_i}"]}} - ) - - # DS4 and DS5 have identical repair consequences - if DS_i == 5: - ds_i = 4 - else: - ds_i = DS_i - - # Convert percentage to ratio. - df_db.loc[(cmp_id, 'Cost'), f'DS{DS_i}-Theta_0'] = ( - f"{S_data['Repair_cost'][occ_type][ds_i - 1] / 100.00:.3f}" - ) - - df_db.loc[(cmp_id, 'Time'), f'DS{DS_i}-Theta_0'] = S_data['Repair_time'][ - occ_type - ][ds_i - 1] - - # store metadata - meta_dict.update({cmp_id: cmp_meta}) - - # Second, the non-structural drift sensitive one - NSD_data = raw_data['NonStructural_Drift_Sensitive_Fragility_Groups'] - - for occ_type in occupancies: - # create the component id - cmp_id = f'NSD.{occ_type}' - - cmp_meta = { - "Description": ( - frag_meta['Meta']['Collections']['NSD']['Description'] - + ", " - + frag_meta['Meta']['OccupancyTypes'][occ_type]['Description'] - ), - "Comments": ( - frag_meta['Meta']['Collections']['NSD']['Comment'] - + "\n" - + frag_meta['Meta']['OccupancyTypes'][occ_type]['Comment'] - ), - "SuggestedComponentBlockSize": "1 EA", - "RoundUpToIntegerQuantity": "True", - "DamageStates": {}, - } - - # store the consequence values for each Damage State - ds_meta = frag_meta['Meta']['Collections']['NSD']['DamageStates'] - for DS_i in range(1, 5): - cmp_meta["DamageStates"].update( - {f"DS{DS_i}": {"Description": ds_meta[f"DS{DS_i}"]}} - ) - - # Convert percentage to ratio. - df_db.loc[(cmp_id, 'Cost'), f'DS{DS_i}-Theta_0'] = ( - f"{NSD_data['Repair_cost'][occ_type][DS_i - 1] / 100.00:.3f}" - ) - - # store metadata - meta_dict.update({cmp_id: cmp_meta}) - - # Third, the non-structural acceleration sensitive fragilities - NSA_data = raw_data['NonStructural_Acceleration_Sensitive_Fragility_Groups'] - - for occ_type in occupancies: - # create the component id - cmp_id = f'NSA.{occ_type}' - - cmp_meta = { - "Description": ( - frag_meta['Meta']['Collections']['NSA']['Description'] - + ", " - + frag_meta['Meta']['OccupancyTypes'][occ_type]['Description'] - ), - "Comments": ( - frag_meta['Meta']['Collections']['NSA']['Comment'] - + "\n" - + frag_meta['Meta']['OccupancyTypes'][occ_type]['Comment'] - ), - "SuggestedComponentBlockSize": "1 EA", - "RoundUpToIntegerQuantity": "True", - "DamageStates": {}, - } - - # store the consequence values for each Damage State - ds_meta = frag_meta['Meta']['Collections']['NSA']['DamageStates'] - for DS_i in range(1, 5): - cmp_meta["DamageStates"].update( - {f"DS{DS_i}": {"Description": ds_meta[f"DS{DS_i}"]}} - ) - - # Convert percentage to ratio. - df_db.loc[(cmp_id, 'Cost'), f'DS{DS_i}-Theta_0'] = ( - f"{NSA_data['Repair_cost'][occ_type][DS_i - 1] / 100.00:.3f}" - ) - - # store metadata - meta_dict.update({cmp_id: cmp_meta}) - - # Fourth, the lifeline facilities - only at the building-level resolution - if resolution == 'building': - LF_data = raw_data['Lifeline_Facilities'] - - for occ_type in occupancies: - # create the component id - cmp_id = f'LF.{occ_type}' - - cmp_meta = { - "Description": ( - frag_meta['Meta']['Collections']['LF']['Description'] - + ", " - + frag_meta['Meta']['OccupancyTypes'][occ_type]['Description'] - ), - "Comments": ( - frag_meta['Meta']['Collections']['LF']['Comment'] - + "\n" - + frag_meta['Meta']['OccupancyTypes'][occ_type]['Comment'] - ), - "SuggestedComponentBlockSize": "1 EA", - "RoundUpToIntegerQuantity": "True", - "DamageStates": {}, - } - - # store the consequence values for each Damage State - ds_meta = frag_meta['Meta']['Collections']['LF']['DamageStates'] - for DS_i in range(1, 6): - # DS4 and DS5 have identical repair consequences - if DS_i == 5: - ds_i = 4 - else: - ds_i = DS_i - - cmp_meta["DamageStates"].update( - {f"DS{DS_i}": {"Description": ds_meta[f"DS{DS_i}"]}} - ) - - # Convert percentage to ratio. - df_db.loc[(cmp_id, 'Cost'), f'DS{DS_i}-Theta_0'] = ( - f"{LF_data['Repair_cost'][occ_type][ds_i - 1] / 100.00:.3f}" - ) - - df_db.loc[(cmp_id, 'Time'), f'DS{DS_i}-Theta_0'] = LF_data[ - 'Repair_time' - ][occ_type][ds_i - 1] - - # store metadata - meta_dict.update({cmp_id: cmp_meta}) - - # remove empty rows (from the end) - df_db.dropna(how='all', inplace=True) - - # All Hazus components have complete fragility info, - df_db['Incomplete'] = 0 - # df_db.loc[:, 'Incomplete'] = 0 - - # The damage quantity unit is the same for all consequence values - df_db.loc[:, 'Quantity-Unit'] = "1 EA" - - # The output units are also identical among all components - df_db.loc[idx[:, 'Cost'], 'DV-Unit'] = "loss_ratio" - df_db.loc[idx[:, 'Time'], 'DV-Unit'] = "day" - - # convert to simple index - df_db = base.convert_to_SimpleIndex(df_db, 0) - - # rename the index - df_db.index.name = "ID" - - # convert to optimal datatypes to reduce file size - df_db = df_db.convert_dtypes() - - # save the consequence data - df_db.to_csv(target_data_file) - - # save the metadata - later - with open(target_meta_file, 'w+', encoding='utf-8') as f: - json.dump(meta_dict, f, indent=2) - - print("Successfully parsed and saved the repair consequence data from Hazus EQ") - - -def create_Hazus_HU_fragility_db( - source_file: str = ( - 'pelicun/resources/SimCenterDBDL/' 'damage_DB_SimCenter_Hazus_HU_bldg.csv' - ), - meta_file: str = ( - 'pelicun/resources/SimCenterDBDL/' - 'damage_DB_SimCenter_Hazus_HU_bldg_template.json' - ), - target_meta_file: str = 'damage_DB_SimCenter_Hazus_HU_bldg.json', -): - """ - Create a database metadata file for the HAZUS Hurricane fragilities. - - This method was developed to add a json file with metadata - accompanying `damage_DB_SimCenter_Hazus_HU_bldg.csv`. That file - contains fragility curves fitted to Hazus Hurricane data related - to the Hazus Hurricane Technical Manual v4.2. - - Parameters - ---------- - source_file: string - Path to the Hazus Hurricane fragility data. - meta_file: string - Path to a predefined fragility metadata file. - target_meta_file: string - Path where the fragility metadata should be saved. A json file is - expected. - - """ - - fragility_data = pd.read_csv(source_file) - - with open(meta_file, 'r', encoding='utf-8') as f: - meta_dict = json.load(f) - - # retrieve damage state descriptions and remove that part from - # `hazus_hu_metadata` - damage_state_classes = meta_dict.pop('DamageStateClasses') - damage_state_descriptions = meta_dict.pop('DamageStateDescriptions') - - # Procedure Overview: - # (1) We define several dictionaries mapping chunks of the - # composite asset ID (the parts between periods) to human-readable - # (`-h` for short) representations. - # (2) We define -h asset type descriptions and map them to the - # first-most relevant ID chunks (`primary chunks`) - # (3) We map asset class codes with general asset classes - # (4) We define the required dictionaries from (1) that decode the - # ID chunks after the `primary chunks` for each general asset - # class - # (5) We decode: - # ID -> asset class -> general asset class -> dictionaries - # -> ID turns to -h text by combining the description of the asset class - # from the `primary chunks` and the decoded description of the - # following chunks using the dictionaries. - - # - # (1) Dictionaries - # - - roof_shape = { - 'flt': 'Flat roof.', - 'gab': 'Gable roof.', - 'hip': 'Hip roof.', - } - - secondary_water_resistance = { - '1': 'Secondary water resistance.', - '0': 'No secondary water resistance.', - 'null': 'No information on secondary water resistance.', - } - - roof_deck_attachment = { - '6d': '6d roof deck nails.', - '6s': '6s roof deck nails.', - '8d': '8d roof deck nails.', - '8s': '8s roof deck nails.', - 'st': 'Standard roof deck attachment.', - 'su': 'Superior roof deck attachment.', - 'null': 'Missing roof deck attachment information.', - } - - roof_wall_connection = { - 'tnail': 'Roof-to-wall toe nails.', - 'strap': 'Roof-to-wall straps.', - 'null': 'Missing roof-to-wall connection information.', - } - - garage_presence = { - 'no': 'No garage.', - 'wkd': 'Weak garage door.', - 'std': 'Standard garage door.', - 'sup': 'Strong garage door.', - 'null': 'No information on garage.', - } - - shutters = {'1': 'Has Shutters.', '0': 'No shutters.'} - - roof_cover = { - 'bur': 'Built-up roof cover.', - 'spm': 'Single-ply membrane roof cover.', - 'smtl': 'Sheet metal roof cover.', - 'cshl': 'Shingle roof cover.', - 'null': 'No information on roof cover.', - } - - roof_quality = { - 'god': 'Good roof quality.', - 'por': 'Poor roof quality.', - 'null': 'No information on roof quality.', - } - - masonry_reinforcing = { - '1': 'Has masonry reinforcing.', - '0': 'No masonry reinforcing.', - 'null': 'Unknown information on masonry reinforcing.', - } - - roof_frame_type = { - 'trs': 'Wood truss roof frame.', - 'ows': 'OWSJ roof frame.', - } - - wind_debris_environment = { - 'A': 'Residential/commercial wind debris environment.', - 'B': 'Wind debris environment varies by direction.', - 'C': 'Residential wind debris environment.', - 'D': 'No wind debris environment.', - } - - roof_deck_age = { - 'god': 'New or average roof age.', - 'por': 'Old roof age.', - 'null': 'Missing roof age information.', - } - - roof_metal_deck_attachment_quality = { - 'std': 'Standard metal deck roof attachment.', - 'sup': 'Superior metal deck roof attachment.', - 'null': 'Missing roof attachment quality information.', - } - - number_of_units = { - 'sgl': 'Single unit.', - 'mlt': 'Multi-unit.', - 'null': 'Unknown number of units.', - } - - joist_spacing = { - '4': '4 ft joist spacing.', - '6': '6 ft foot joist spacing.', - 'null': 'Unknown joist spacing.', - } - - window_area = { - 'low': 'Low window area.', - 'med': 'Medium window area.', - 'hig': 'High window area.', - } - - tie_downs = {'1': 'Tie downs.', '0': 'No tie downs.'} - - terrain_surface_roughness = { - '3': 'Terrain surface roughness: 0.03 m.', - '15': 'Terrain surface roughness: 0.15 m.', - '35': 'Terrain surface roughness: 0.35 m.', - '70': 'Terrain surface roughness: 0.7 m.', - '100': 'Terrain surface roughness: 1 m.', - } - - # - # (2) Asset type descriptions - # - - # maps class type code to -h description - class_types = { - # ------------------------ - 'W.SF.1': 'Wood, Single-family, One-story.', - 'W.SF.2': 'Wood, Single-family, Two or More Stories.', - # ------------------------ - 'W.MUH.1': 'Wood, Multi-Unit Housing, One-story.', - 'W.MUH.2': 'Wood, Multi-Unit Housing, Two Stories.', - 'W.MUH.3': 'Wood, Multi-Unit Housing, Three or More Stories.', - # ------------------------ - 'M.SF.1': 'Masonry, Single-family, One-story.', - 'M.SF.2': 'Masonry, Single-family, Two or More Stories.', - # ------------------------ - 'M.MUH.1': 'Masonry, Multi-Unit Housing, One-story.', - 'M.MUH.2': 'Masonry, Multi-Unit Housing, Two Stories.', - 'M.MUH.3': 'Masonry, Multi-Unit Housing, Three or More Stories.', - # ------------------------ - 'M.LRM.1': 'Masonry, Low-Rise Strip Mall, Up to 15 Feet.', - 'M.LRM.2': 'Masonry, Low-Rise Strip Mall, More than 15 Feet.', - # ------------------------ - 'M.LRI': 'Masonry, Low-Rise Industrial/Warehouse/Factory Buildings.', - # ------------------------ - 'M.ERB.L': ( - 'Masonry, Engineered Residential Building, Low-Rise (1-2 Stories).' - ), - 'M.ERB.M': ( - 'Masonry, Engineered Residential Building, Mid-Rise (3-5 Stories).' - ), - 'M.ERB.H': ( - 'Masonry, Engineered Residential Building, High-Rise (6+ Stories).' - ), - # ------------------------ - 'M.ECB.L': ( - 'Masonry, Engineered Commercial Building, Low-Rise (1-2 Stories).' - ), - 'M.ECB.M': ( - 'Masonry, Engineered Commercial Building, Mid-Rise (3-5 Stories).' - ), - 'M.ECB.H': ( - 'Masonry, Engineered Commercial Building, High-Rise (6+ Stories).' - ), - # ------------------------ - 'C.ERB.L': ( - 'Concrete, Engineered Residential Building, Low-Rise (1-2 Stories).' - ), - 'C.ERB.M': ( - 'Concrete, Engineered Residential Building, Mid-Rise (3-5 Stories).' - ), - 'C.ERB.H': ( - 'Concrete, Engineered Residential Building, High-Rise (6+ Stories).' - ), - # ------------------------ - 'C.ECB.L': ( - 'Concrete, Engineered Commercial Building, Low-Rise (1-2 Stories).' - ), - 'C.ECB.M': ( - 'Concrete, Engineered Commercial Building, Mid-Rise (3-5 Stories).' - ), - 'C.ECB.H': ( - 'Concrete, Engineered Commercial Building, High-Rise (6+ Stories).' - ), - # ------------------------ - 'S.PMB.S': 'Steel, Pre-Engineered Metal Building, Small.', - 'S.PMB.M': 'Steel, Pre-Engineered Metal Building, Medium.', - 'S.PMB.L': 'Steel, Pre-Engineered Metal Building, Large.', - # ------------------------ - 'S.ERB.L': 'Steel, Engineered Residential Building, Low-Rise (1-2 Stories).', - 'S.ERB.M': 'Steel, Engineered Residential Building, Mid-Rise (3-5 Stories).', - 'S.ERB.H': 'Steel, Engineered Residential Building, High-Rise (6+ Stories).', - # ------------------------ - 'S.ECB.L': 'Steel, Engineered Commercial Building, Low-Rise (1-2 Stories).', - 'S.ECB.M': 'Steel, Engineered Commercial Building, Mid-Rise (3-5 Stories).', - 'S.ECB.H': 'Steel, Engineered Commercial Building, High-Rise (6+ Stories).', - # ------------------------ - 'MH.PHUD': 'Manufactured Home, Pre-Housing and Urban Development (HUD).', - 'MH.76HUD': 'Manufactured Home, 1976 HUD.', - 'MH.94HUDI': 'Manufactured Home, 1994 HUD - Wind Zone I.', - 'MH.94HUDII': 'Manufactured Home, 1994 HUD - Wind Zone II.', - 'MH.94HUDIII': 'Manufactured Home, 1994 HUD - Wind Zone III.', - # ------------------------ - 'HUEF.H.S': 'Small Hospital, Hospital with fewer than 50 Beds.', - 'HUEF.H.M': 'Medium Hospital, Hospital with beds between 50 & 150.', - 'HUEF.H.L': 'Large Hospital, Hospital with more than 150 Beds.', - # ------------------------ - 'HUEF.S.S': 'Elementary School.', - 'HUEF.S.M': 'High school, two-story.', - 'HUEF.S.L': 'Large high school, three-story.', - # ------------------------ - 'HUEF.EO': 'Emergency Operation Centers.', - 'HUEF.FS': 'Fire Station.', - 'HUEF.PS': 'Police Station.', - # ------------------------ - } - - def find_class_type(entry: str) -> str | None: - """ - Find the class type code from an entry string based on - predefined patterns. - - Parameters - ---------- - entry : str - A string representing the entry, consisting of delimited - segments that correspond to various attributes of an - asset. - - Returns - ------- - str or None - The class type code if a matching pattern is found; - otherwise, None if no pattern matches the input string. - - """ - entry_elements = entry.split('.') - for nper in range(1, len(entry_elements)): - first_parts = '.'.join(entry_elements[:nper]) - if first_parts in class_types: - return first_parts - return None - - # - # (3) General asset class - # - - # maps class code type to general class code - general_classes = { - # ------------------------ - 'W.SF.1': 'WSF', - 'W.SF.2': 'WSF', - # ------------------------ - 'W.MUH.1': 'WMUH', - 'W.MUH.2': 'WMUH', - 'W.MUH.3': 'WMUH', - # ------------------------ - 'M.SF.1': 'MSF', - 'M.SF.2': 'MSF', - # ------------------------ - 'M.MUH.1': 'MMUH', - 'M.MUH.2': 'MMUH', - 'M.MUH.3': 'MMUH', - # ------------------------ - 'M.LRM.1': 'MLRM1', - 'M.LRM.2': 'MLRM2', - # ------------------------ - 'M.LRI': 'MLRI', - # ------------------------ - 'M.ERB.L': 'MERB', - 'M.ERB.M': 'MERB', - 'M.ERB.H': 'MERB', - # ------------------------ - 'M.ECB.L': 'MECB', - 'M.ECB.M': 'MECB', - 'M.ECB.H': 'MECB', - # ------------------------ - 'C.ERB.L': 'CERB', - 'C.ERB.M': 'CERB', - 'C.ERB.H': 'CERB', - # ------------------------ - 'C.ECB.L': 'CECB', - 'C.ECB.M': 'CECB', - 'C.ECB.H': 'CECB', - # ------------------------ - 'S.PMB.S': 'SPMB', - 'S.PMB.M': 'SPMB', - 'S.PMB.L': 'SPMB', - # ------------------------ - 'S.ERB.L': 'SERB', - 'S.ERB.M': 'SERB', - 'S.ERB.H': 'SERB', - # ------------------------ - 'S.ECB.L': 'SECB', - 'S.ECB.M': 'SECB', - 'S.ECB.H': 'SECB', - # ------------------------ - 'MH.PHUD': 'MH', - 'MH.76HUD': 'MH', - 'MH.94HUDI': 'MH', - 'MH.94HUDII': 'MH', - 'MH.94HUDIII': 'MH', - # ------------------------ - 'HUEF.H.S': 'HUEFH', - 'HUEF.H.M': 'HUEFH', - 'HUEF.H.L': 'HUEFH', - # ------------------------ - 'HUEF.S.S': 'HUEFS', - 'HUEF.S.M': 'HUEFS', - 'HUEF.S.L': 'HUEFS', - # ------------------------ - 'HUEF.EO': 'HUEFEO', - 'HUEF.FS': 'HUEFFS', - 'HUEF.PS': 'HUEFPS', - # ------------------------ - } - - # - # (4) Relevant dictionaries - # - - # maps general class code to list of dicts where the -h attribute - # descriptions will be pulled from - dictionaries_of_interest = { - 'WSF': [ - roof_shape, - secondary_water_resistance, - roof_deck_attachment, - roof_wall_connection, - garage_presence, - shutters, - terrain_surface_roughness, - ], - 'WMUH': [ - roof_shape, - roof_cover, - roof_quality, - secondary_water_resistance, - roof_deck_attachment, - roof_wall_connection, - shutters, - terrain_surface_roughness, - ], - 'MSF': [ - roof_shape, - roof_wall_connection, - roof_frame_type, - roof_deck_attachment, - shutters, - secondary_water_resistance, - garage_presence, - masonry_reinforcing, - roof_cover, - terrain_surface_roughness, - ], - 'MMUH': [ - roof_shape, - secondary_water_resistance, - roof_cover, - roof_quality, - roof_deck_attachment, - roof_wall_connection, - shutters, - masonry_reinforcing, - terrain_surface_roughness, - ], - 'MLRM1': [ - roof_cover, - shutters, - masonry_reinforcing, - wind_debris_environment, - roof_frame_type, - roof_deck_attachment, - roof_wall_connection, - roof_deck_age, - roof_metal_deck_attachment_quality, - terrain_surface_roughness, - ], - 'MLRM2': [ - roof_cover, - shutters, - masonry_reinforcing, - wind_debris_environment, - roof_frame_type, - roof_deck_attachment, - roof_wall_connection, - roof_deck_age, - roof_metal_deck_attachment_quality, - number_of_units, - joist_spacing, - terrain_surface_roughness, - ], - 'MLRI': [ - shutters, - masonry_reinforcing, - roof_deck_age, - roof_metal_deck_attachment_quality, - terrain_surface_roughness, - ], - 'MERB': [ - roof_cover, - shutters, - wind_debris_environment, - roof_metal_deck_attachment_quality, - window_area, - terrain_surface_roughness, - ], - 'MECB': [ - roof_cover, - shutters, - wind_debris_environment, - roof_metal_deck_attachment_quality, - window_area, - terrain_surface_roughness, - ], - 'CERB': [ - roof_cover, - shutters, - wind_debris_environment, - window_area, - terrain_surface_roughness, - ], - 'CECB': [ - roof_cover, - shutters, - wind_debris_environment, - window_area, - terrain_surface_roughness, - ], - 'SPMB': [ - shutters, - roof_deck_age, - roof_metal_deck_attachment_quality, - terrain_surface_roughness, - ], - 'SERB': [ - roof_cover, - shutters, - wind_debris_environment, - roof_metal_deck_attachment_quality, - window_area, - terrain_surface_roughness, - ], - 'SECB': [ - roof_cover, - shutters, - wind_debris_environment, - roof_metal_deck_attachment_quality, - window_area, - terrain_surface_roughness, - ], - 'MH': [shutters, tie_downs, terrain_surface_roughness], - 'HUEFH': [ - roof_cover, - wind_debris_environment, - roof_metal_deck_attachment_quality, - shutters, - terrain_surface_roughness, - ], - 'HUEFS': [ - roof_cover, - shutters, - wind_debris_environment, - roof_deck_age, - roof_metal_deck_attachment_quality, - terrain_surface_roughness, - ], - 'HUEFEO': [ - roof_cover, - shutters, - wind_debris_environment, - roof_metal_deck_attachment_quality, - window_area, - terrain_surface_roughness, - ], - 'HUEFFS': [ - roof_cover, - shutters, - wind_debris_environment, - roof_deck_age, - roof_metal_deck_attachment_quality, - terrain_surface_roughness, - ], - 'HUEFPS': [ - roof_cover, - shutters, - wind_debris_environment, - roof_metal_deck_attachment_quality, - window_area, - terrain_surface_roughness, - ], - } - - # - # (5) Decode IDs and extend metadata with the individual records - # - - for fragility_id in fragility_data['ID'].to_list(): - class_type = find_class_type(fragility_id) - assert class_type is not None - - class_type_human_readable = class_types[class_type] - - general_class = general_classes[class_type] - dictionaries = dictionaries_of_interest[general_class] - remaining_chunks = fragility_id.replace(f'{class_type}.', '').split('.') - assert len(remaining_chunks) == len(dictionaries) - human_description = [class_type_human_readable] - for chunk, dictionary in zip(remaining_chunks, dictionaries): - human_description.append(dictionary[chunk]) - human_description_str = ' '.join(human_description) - - damage_state_class = damage_state_classes[class_type] - damage_state_description = damage_state_descriptions[damage_state_class] - - limit_states = {} - for damage_state, description in damage_state_description.items(): - limit_state = damage_state.replace('DS', 'LS') - limit_states[limit_state] = {damage_state: description} - - record = { - 'Description': human_description_str, - 'SuggestedComponentBlockSize': '1 EA', - 'RoundUpToIntegerQuantity': 'True', - 'LimitStates': limit_states, - } - - meta_dict[fragility_id] = record - - # save the metadata - with open(target_meta_file, 'w+', encoding='utf-8') as f: - json.dump(meta_dict, f, indent=2) - - -def create_Hazus_Flood_repair_db(source_file_dir: str, target_data_file: str): - """ - Create a database metadata file for the HAZUS Flood - loss functions. - - Parameters - ---------- - source_file_dir: string - Path to the directory containing Hazus Flood loss function - data files. - target_data_file: string - Path where the loss function data should be saved. A CSV file - is expected. - - """ - - source_data = {} - for subassembly_type in ('structural', 'inventory', 'contents'): - source_file = ( - f'{source_file_dir}/HazusFloodDamageFunctions_' - f'Hazus61_{subassembly_type}.csv' - ) - source_data[subassembly_type] = pd.read_csv(source_file) - - def eda(): - """ - Exploratory data analysis code. - """ - - subassembly_type = 'contents' - df = source_data[subassembly_type] - - # General sense - print(df.shape) - print(df.info()) - print(df.head()) - print(df.describe()) - - # Unique categorical values - categorical_columns = ('Default', 'Occupancy', 'Source') - for column in categorical_columns: - print(f"Unique values in {column}: {df[column].unique()}") - - # Histograms for numerical columns - numerical_columns = df.select_dtypes(include=['float64', 'int64']).columns - df[numerical_columns].hist(figsize=(15, 10), bins=30) - plt.show() - - # Bar charts of categorical data - for column in categorical_columns: - df[column].value_counts().plot(kind='bar') - plt.title(column) - plt.show() - - # Some `contents` loss functions are on inventory, while there is - # a specific `inventory` subassembly. - print( - source_data['contents']['Description'][ - source_data['contents']['Description'].str.contains('entory') - ] - ) - - # We have a dedicated column for `subassembly`, so we don't need - # special names for the function ID for each subassembly set. - source_data['structural'] = source_data['structural'].rename( - columns={'BldgDmgFnID': 'FnID'} - ) - source_data['inventory'] = source_data['inventory'].rename( - columns={'InvDmgFnId': 'FnID'} - ) - source_data['contents'] = source_data['contents'].rename( - columns={'ContDmgFnId': 'FnID'} - ) - - # Merge the three subassembly datasets - df = pd.concat(source_data.values(), keys=source_data.keys(), axis=0) - df.index.names = ('subassembly', 'index') - - # Columns defining the loss for each inundation height - ft_cols = [] - for col in df.columns: - if col.startswith('ft'): - ft_cols.append(col) - ft_values_list = [] - for x in ft_cols: - if 'm' in x: - ft_values_list.append(-float(x.replace('m', '').replace('ft', ''))) - else: - ft_values_list.append(float(x.replace('ft', ''))) - ft_values = np.array(ft_values_list) - - unique_sources = df['Source'].unique() - source_map = {} - for source in unique_sources: - source_value = ( - source.replace('(MOD.)', 'Modified') - .replace(' - ', '_') - .replace('.', '') - .replace(' ', '-') - ) - source_map[source] = source_value - - lf_data = pd.DataFrame( - index=df.index.get_level_values('index'), - columns=[ - 'ID', - 'Incomplete', - 'Demand-Type', - 'Demand-Unit', - 'Demand-Offset', - 'Demand-Directional', - 'DV-Unit', - 'LossFunction-Theta_0', - ], - ) - # assign common values - lf_data['Incomplete'] = 0 - lf_data['Demand-Type'] = 'Peak Inundation Height' - lf_data['Demand-Unit'] = 'in' - lf_data['Demand-Offset'] = 0 - lf_data['Demand-Directional'] = 1 - lf_data['DV-Unit'] = 'loss_ratio' - - def remove_repeated_chars(s): - """ - Removes all repeated instances of a character in a string with - a single instance of that character. - - Parameters - ---------- - s : str - The input string. - - Returns - ------- - str - The string with repeated instances of characters replaced - by a single instance. - - """ - - if not s: - return "" - - result = [s[0]] # Initialize result with the first character - - for char in s[1:]: - if char.isalpha() or char != result[-1]: - result.append(char) - - return ''.join(result) - - for index, row in df.iterrows(): - - # Extract row data - data_type = index[0] - row_index = index[1] - occupancy = row.Occupancy.strip() - lf_id = row.FnID - source = source_map[row.Source] - description = row.Description - - # loss function information - ys = ', '.join([str(x) for x in row[ft_cols].to_list()]) - xs = ', '.join([str(x) for x in ft_values.tolist()]) - lf_str = f'{ys}|{xs}' - - lf_data.loc[row_index, 'LossFunction-Theta_0'] = lf_str - - # assign an ID - lf_id = '.'.join([occupancy, source, data_type]) - - other_data = ( - description.lower() - .replace('contents', '') - .replace('(equipment)', 'equipment') - .replace('(inventory)', 'inventory') - .replace('(equipment/inventory)', 'equipment/inventory') - .replace('(inventory/equipment)', 'equipment/inventory') - .replace(':', '') - .replace('(', '') - .replace(')', '') - .split(',') - ) - for other in other_data: - other = other.strip() - if not other: - continue - elements = [ - x.replace('-', '_').replace('w/', 'with') for x in other.split(' ') - ] - if elements: - lf_id += '.' + '_'.join(elements) - lf_id += '-Cost' - - lf_data.loc[row_index, 'ID'] = lf_id - - lf_data['ID'] = lf_data['ID'].apply(remove_repeated_chars) - - lf_data.to_csv(target_data_file) diff --git a/pelicun/file_io.py b/pelicun/file_io.py index 403e70298..f58a6b61f 100644 --- a/pelicun/file_io.py +++ b/pelicun/file_io.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -40,26 +39,16 @@ # Kuanshi Zhong # John Vouvakis Manousakis -""" -This module has classes and methods that handle file input and output. - -.. rubric:: Contents - -.. autosummary:: - - get_required_resources - save_to_csv - load_data - load_from_file - -""" +"""Classes and methods that handle file input and output.""" from __future__ import annotations + from pathlib import Path + import numpy as np import pandas as pd -from pelicun import base +from pelicun import base convert_dv_name = { 'DV_rec_cost': 'Reconstruction Cost', @@ -92,17 +81,18 @@ } -def save_to_csv( - data: pd.DataFrame, - filepath_str: str | None, +def save_to_csv( # noqa: C901 + data: pd.DataFrame | None, + filepath: Path | None, units: pd.Series | None = None, unit_conversion_factors: dict | None = None, orientation: int = 0, + *, use_simpleindex: bool = True, log: base.Logger | None = None, ) -> pd.DataFrame | None: """ - Saves data to a CSV file following the standard SimCenter schema. + Save data to a CSV file following the standard SimCenter schema. The produced CSV files have a single header line and an index column. The second line may start with 'Units' in the index or the @@ -111,25 +101,25 @@ def save_to_csv( Parameters ---------- - data : DataFrame + data: DataFrame The data to save. - filepath : str + filepath: Path The location of the destination file. If None, the data is not saved, but returned in the end. - units : Series, optional + units: Series, optional Provides a Series with variables and corresponding units. - unit_conversion_factors : dict, optional + unit_conversion_factors: dict, optional Dictionary containing key-value pairs of unit names and their corresponding factors. Conversion factors are defined as the number of times a base unit fits in the alternative unit. - orientation : int, {0, 1}, default 0 + orientation: int, {0, 1}, default 0 If 0, variables are organized along columns; otherwise, they are along the rows. This is important when converting values to follow the prescribed units. - use_simpleindex : bool, default True + use_simpleindex: bool, default True If True, MultiIndex columns and indexes are converted to SimpleIndex before saving. - log : Logger, optional + log: Logger, optional Logger object to be used. If no object is specified, no logging is performed. @@ -148,45 +138,46 @@ def save_to_csv( If `filepath` is None, returns the DataFrame with potential unit conversions and reformatting applied. Otherwise, returns None after saving the data to a CSV file. - """ - if filepath_str is None: + """ + if filepath is None: if log: log.msg('Preparing data ...', prepend_timestamp=False) elif log: - log.msg(f'Saving data to `{filepath_str}`...', prepend_timestamp=False) + log.msg(f'Saving data to `{filepath!s}`...', prepend_timestamp=False) if data is None: if log: - log.warn('Data was empty, no file saved.') + log.warning('Data was empty, no file saved.') return None + assert isinstance(data, pd.DataFrame) + # make sure we do not modify the original data data = data.copy() # convert units and add unit information, if needed if units is not None: - if unit_conversion_factors is None: - raise ValueError( + msg = ( 'When `units` is not None, ' '`unit_conversion_factors` must be provided.' ) + raise ValueError(msg) if log: log.msg('Converting units...', prepend_timestamp=False) # if the orientation is 1, we might not need to scale all columns if orientation == 1: - cols_to_scale_bool = [dt in [float, int] for dt in data.dtypes] + cols_to_scale_bool = [dt in {float, int} for dt in data.dtypes] cols_to_scale = data.columns[cols_to_scale_bool] labels_to_keep = [] for unit_name in units.unique(): - - labels = units.loc[units == unit_name].index.values + labels = units.loc[units == unit_name].index.to_numpy() unit_factor = 1.0 / unit_conversion_factors[unit_name] @@ -195,7 +186,7 @@ def save_to_csv( if orientation == 0: for label in labels: if label in data.columns: - active_labels.append(label) + active_labels.append(label) # noqa: PERF401 if len(active_labels) > 0: data.loc[:, active_labels] *= unit_factor @@ -203,12 +194,12 @@ def save_to_csv( else: # elif orientation == 1: for label in labels: if label in data.index: - active_labels.append(label) + active_labels.append(label) # noqa: PERF401 if len(active_labels) > 0: - data.loc[ - np.array(active_labels), np.array(cols_to_scale) - ] *= unit_factor + data.loc[np.array(active_labels), np.array(cols_to_scale)] *= ( + unit_factor + ) labels_to_keep += active_labels @@ -216,14 +207,15 @@ def save_to_csv( if orientation == 0: data = pd.concat([units_df.T, data], axis=0) - data.sort_index(axis=1, inplace=True) + data = data.sort_index(axis=1) else: data = pd.concat([units_df, data], axis=1) - data.sort_index(inplace=True) + data = data.sort_index() if log: log.msg('Unit conversion successful.', prepend_timestamp=False) + assert isinstance(data, pd.DataFrame) if use_simpleindex: # convert MultiIndex to regular index with '-' separators if isinstance(data.index, pd.MultiIndex): @@ -233,11 +225,8 @@ def save_to_csv( if isinstance(data.columns, pd.MultiIndex): data = base.convert_to_SimpleIndex(data, axis=1) - if filepath_str is not None: - - filepath = Path(filepath_str).resolve() + if filepath is not None: if filepath.suffix == '.csv': - # save the contents of the DataFrame into a csv data.to_csv(filepath) @@ -245,10 +234,11 @@ def save_to_csv( log.msg('Data successfully saved to file.', prepend_timestamp=False) else: - raise ValueError( - f'ERROR: Please use the `.csv` file extension. ' + msg = ( + f'Please use the `.csv` file extension. ' f'Received file name is `{filepath}`' ) + raise ValueError(msg) return None @@ -256,10 +246,11 @@ def save_to_csv( return data -def substitute_default_path(data_paths: list[str]) -> list[str]: +def substitute_default_path( + data_paths: list[str | pd.DataFrame], +) -> list[str | pd.DataFrame]: """ - Substitutes the default directory path in a list of data paths - with a specified path. + Substitute the default directory path with a specified path. This function iterates over a list of data paths and replaces occurrences of the 'PelicunDefault/' substring with the path @@ -271,7 +262,7 @@ def substitute_default_path(data_paths: list[str]) -> list[str]: Parameters ---------- - data_paths : list of str + data_paths: list of str A list containing the paths to data files. These paths may include a placeholder directory 'PelicunDefault/' that needs to be substituted with the actual path specified in @@ -292,8 +283,8 @@ def substitute_default_path(data_paths: list[str]) -> list[str]: - If a path in the input list does not contain 'PelicunDefault/', it is added to the output list unchanged. - Example - ------- + Examples + -------- >>> data_paths = ['PelicunDefault/data/file1.txt', 'data/file2.txt'] >>> substitute_default_path(data_paths) @@ -301,9 +292,9 @@ def substitute_default_path(data_paths: list[str]) -> list[str]: 'data/file2.txt'] """ - updated_paths = [] + updated_paths: list[str | pd.DataFrame] = [] for data_path in data_paths: - if 'PelicunDefault/' in data_path: + if isinstance(data_path, str) and 'PelicunDefault/' in data_path: path = data_path.replace( 'PelicunDefault/', f'{base.pelicun_path}/resources/SimCenterDBDL/', @@ -314,16 +305,17 @@ def substitute_default_path(data_paths: list[str]) -> list[str]: return updated_paths -def load_data( +def load_data( # noqa: C901 data_source: str | pd.DataFrame, - unit_conversion_factors: dict | None, + unit_conversion_factors: dict | None = None, orientation: int = 0, + *, reindex: bool = True, return_units: bool = False, log: base.Logger | None = None, ) -> tuple[pd.DataFrame, pd.Series] | pd.DataFrame: """ - Loads data assuming it follows standard SimCenter tabular schema. + Load data assuming it follows standard SimCenter tabular schema. The data is assumed to have a single header line and an index column. The second line may start with 'Units' in the index and provide the units for @@ -368,12 +360,8 @@ def load_data( TypeError If `data_source` is neither a string nor a DataFrame, a TypeError is raised. - ValueError - If `unit_conversion_factors` contains keys that do not - correspond to any units in the data, a ValueError may be - raised during processing. - """ + """ if isinstance(data_source, pd.DataFrame): # store it at proceed (copying is needed to avoid changing the # original) @@ -382,7 +370,8 @@ def load_data( # otherwise, load the data from a file data = load_from_file(data_source) else: - raise TypeError(f'Invalid data_source type: {type(data_source)}') + msg = f'Invalid data_source type: {type(data_source)}' + raise TypeError(msg) # Define a dictionary to decide the axis based on the orientation axis = {0: 1, 1: 0} @@ -392,14 +381,16 @@ def load_data( # and optionally apply conversions to all numeric values if 'Units' in the_index: units = data['Units'] if orientation == 1 else data.loc['Units'] - data.drop(['Units'], axis=orientation, inplace=True) # type: ignore + data = data.drop(['Units'], axis=orientation) # type: ignore data = base.convert_dtypes(data) if unit_conversion_factors is not None: numeric_elements = ( - (data.select_dtypes(include=[np.number]).index) + (data.select_dtypes(include=[np.number]).index) # type: ignore if orientation == 0 - else (data.select_dtypes(include=[np.number]).columns) + else ( + data.select_dtypes(include=[np.number]).columns # type: ignore + ) ) if log: @@ -415,15 +406,17 @@ def load_data( if orientation == 1: data.loc[:, numeric_elements] = data.loc[ - :, numeric_elements + :, numeric_elements # type: ignore ].multiply( - conversion_factors, axis=axis[orientation] + conversion_factors, + axis=axis[orientation], # type: ignore ) # type: ignore else: data.loc[numeric_elements, :] = data.loc[ numeric_elements, : ].multiply( - conversion_factors, axis=axis[orientation] + conversion_factors, + axis=axis[orientation], # type: ignore ) # type: ignore if log: @@ -435,7 +428,7 @@ def load_data( # convert columns or index to MultiIndex if needed data = base.convert_to_MultiIndex(data, axis=1) - data.sort_index(axis=1, inplace=True) + data = data.sort_index(axis=1) # reindex the data, if needed if reindex: @@ -443,23 +436,23 @@ def load_data( else: # convert index to MultiIndex if needed data = base.convert_to_MultiIndex(data, axis=0) - data.sort_index(inplace=True) + data = data.sort_index() if return_units: if units is not None: # convert index in units Series to MultiIndex if needed - units = base.convert_to_MultiIndex(units, axis=0).dropna() # type: ignore # noqa - units.sort_index(inplace=True) + units = base.convert_to_MultiIndex(units, axis=0).dropna() # type: ignore + units = units.sort_index() output = data, units else: - output = data + output = data # type: ignore - return output + return output # type: ignore def load_from_file(filepath: str, log: base.Logger | None = None) -> pd.DataFrame: """ - Loads data from a file and stores it in a DataFrame. + Load data from a file and stores it in a DataFrame. Currently, only CSV files are supported, but the function is easily extensible to support other file formats. @@ -467,7 +460,9 @@ def load_from_file(filepath: str, log: base.Logger | None = None) -> pd.DataFram Parameters ---------- filepath: string - The location of the source file + The location of the source file. + log: base.Logger, optional + Optional logger object. Returns ------- @@ -484,8 +479,8 @@ def load_from_file(filepath: str, log: base.Logger | None = None) -> pd.DataFram If the filepath is invalid. ValueError If the file is not a CSV. - """ + """ if log: log.msg(f'Loading data from {filepath}...') @@ -493,10 +488,11 @@ def load_from_file(filepath: str, log: base.Logger | None = None) -> pd.DataFram filepath_path = Path(filepath).resolve() if not filepath_path.is_file(): - raise FileNotFoundError( - f"The filepath provided does not point to an existing " - f"file: {filepath_path}" + msg = ( + f'The filepath provided does not point to an existing ' + f'file: {filepath_path}' ) + raise FileNotFoundError(msg) if filepath_path.suffix == '.csv': # load the contents of the csv into a DataFrame @@ -513,9 +509,10 @@ def load_from_file(filepath: str, log: base.Logger | None = None) -> pd.DataFram log.msg('File successfully opened.', prepend_timestamp=False) else: - raise ValueError( - f'ERROR: Unexpected file type received when trying ' + msg = ( + f'Unexpected file type received when trying ' f'to load from csv: {filepath_path}' ) + raise ValueError(msg) return data diff --git a/pelicun/model/__init__.py b/pelicun/model/__init__.py index f4734f486..41aa1fa1f 100644 --- a/pelicun/model/__init__.py +++ b/pelicun/model/__init__.py @@ -1,52 +1,48 @@ -""" --*- coding: utf-8 -*- +# +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California -Copyright (c) 2018 Leland Stanford Junior University -Copyright (c) 2018 The Regents of the University of California +# This file is part of pelicun. -This file is part of pelicun. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. -1. Redistributions of source code must retain the above copyright notice, -this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. -3. Neither the name of the copyright holder nor the names of its contributors -may be used to endorse or promote products derived from this software without -specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . -You should have received a copy of the BSD 3-Clause License along with -pelicun. If not, see . - -Contributors: -Adam Zsarnóczay -""" - -# flake8: noqa +"""Pelicun models.""" from __future__ import annotations -from pelicun.model.pelicun_model import PelicunModel -from pelicun.model.demand_model import DemandModel + from pelicun.model.asset_model import AssetModel -from pelicun.model.damage_model import DamageModel -from pelicun.model.damage_model import DamageModel_DS -from pelicun.model.loss_model import LossModel -from pelicun.model.loss_model import RepairModel_DS -from pelicun.model.loss_model import RepairModel_LF +from pelicun.model.damage_model import DamageModel, DamageModel_DS +from pelicun.model.demand_model import DemandModel +from pelicun.model.loss_model import ( + LossModel, + RepairModel_DS, + RepairModel_LF, +) +from pelicun.model.pelicun_model import PelicunModel diff --git a/pelicun/model/asset_model.py b/pelicun/model/asset_model.py index a777bb81c..7fafa8c84 100644 --- a/pelicun/model/asset_model.py +++ b/pelicun/model/asset_model.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -38,75 +37,73 @@ # Adam Zsarnóczay # John Vouvakis Manousakis -""" -This file defines the AssetModel object and its methods. -.. rubric:: Contents - -.. autosummary:: - - AssetModel - -""" +"""AssetModel object and methods.""" from __future__ import annotations -from typing import TYPE_CHECKING + from itertools import product +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable + import numpy as np import pandas as pd + +from pelicun import base, file_io, uq from pelicun.model.pelicun_model import PelicunModel -from pelicun import base -from pelicun import uq -from pelicun import file_io if TYPE_CHECKING: - from pelicun.assessment import Assessment + from pelicun.assessment import AssessmentBase idx = base.idx class AssetModel(PelicunModel): - """ - Manages asset information used in assessments. + """Asset information used in assessments.""" - Parameters - ---------- + __slots__ = ['_cmp_RVs', 'cmp_marginal_params', 'cmp_sample', 'cmp_units'] - """ + def __init__(self, assessment: AssessmentBase) -> None: + """ + Initialize an Asset model. - __slots__ = ['cmp_marginal_params', 'cmp_units', 'cmp_sample', '_cmp_RVs'] + Parameters + ---------- + assessment: AssessmentBase + Parent assessment object. - def __init__(self, assessment: Assessment): + """ super().__init__(assessment) - self.cmp_marginal_params = None - self.cmp_units = None - self.cmp_sample = None + self.cmp_marginal_params: pd.DataFrame | None = None + self.cmp_units: pd.Series | None = None + self.cmp_sample: pd.DataFrame | None = None - self._cmp_RVs = None + self._cmp_RVs: uq.RandomVariableRegistry | None = None def save_cmp_sample( - self, filepath: str | None = None, save_units: bool = False + self, filepath: str | None = None, *, save_units: bool = False ) -> pd.DataFrame | tuple[pd.DataFrame, pd.Series] | None: """ - Saves the component quantity sample to a CSV file or returns - it as a DataFrame with optional units. + Save or retrieve component quantity sample. - This method handles the storage of a sample of component - quantities, which can either be saved directly to a file or - returned as a DataFrame for further manipulation. When saving - to a file, additional information such as unit conversion - factors and column units can be included. If the data is not - being saved to a file, the method can return the DataFrame - with or without units as specified. + Saves the component quantity sample to a CSV file or returns + it as a DataFrame with optional units. This method handles + the storage of a sample of component quantities, which can + either be saved directly to a file or returned as a DataFrame + for further manipulation. When saving to a file, additional + information such as unit conversion factors and column units + can be included. If the data is not being saved to a file, the + method can return the DataFrame with or without units as + specified. Parameters ---------- - filepath : str, optional + filepath: str, optional The path to the file where the component quantity sample should be saved. If not provided, the sample is not saved to disk but returned. - save_units : bool, default: False + save_units: bool, default: False Indicates whether to include a row with unit information in the returned DataFrame. This parameter is ignored if a file path is provided. @@ -121,17 +118,12 @@ def save_cmp_sample( * Optionally, a Series containing the units for each column if `save_units` is True. - Raises - ------ - IOError - Raises an IOError if there is an issue saving the file to - the specified `filepath`. - Notes ----- The function utilizes internal logging to notify the start and completion of the saving process. It adjusts index types and handles unit conversions based on assessment configurations. + """ self.log.div() if filepath is not None: @@ -139,21 +131,22 @@ def save_cmp_sample( # prepare a units array sample = self.cmp_sample + assert isinstance(sample, pd.DataFrame) units = pd.Series(name='Units', index=sample.columns, dtype=object) + assert self.cmp_units is not None for cmp_id, unit_name in self.cmp_units.items(): - units.loc[cmp_id, :] = unit_name + units.loc[cmp_id, :] = unit_name # type: ignore res = file_io.save_to_csv( sample, - filepath, + Path(filepath) if filepath is not None else None, units=units, unit_conversion_factors=self._asmnt.unit_conversion_factors, use_simpleindex=(filepath is not None), log=self._asmnt.log, ) - if filepath is not None: self.log.msg( 'Asset components sample successfully saved.', @@ -161,8 +154,14 @@ def save_cmp_sample( ) return None # else: - units = res.loc["Units"] - res.drop("Units", inplace=True) + + assert isinstance(res, pd.DataFrame) + + units_part = res.loc['Units'] + assert isinstance(units_part, pd.Series) + units = units_part + + res = res.drop('Units') if save_units: return res.astype(float), units @@ -171,8 +170,7 @@ def save_cmp_sample( def load_cmp_sample(self, filepath: str) -> None: """ - Loads a component quantity sample from a specified CSV file - into the system. + Load a component quantity sample from a specified CSV file. This method reads a CSV file that contains component quantity samples, setting up the necessary DataFrame structures within @@ -182,10 +180,15 @@ def load_cmp_sample(self, filepath: str) -> None: Parameters ---------- - filepath : str + filepath: str The path to the CSV file from which to load the component quantity sample. + Raises + ------ + ValueError + If the columns have an invalid number of levels. + Notes ----- Upon successful loading, the method sets the component sample @@ -203,6 +206,7 @@ def load_cmp_sample(self, filepath: str) -> None: >>> model.load_cmp_sample('path/to/component_sample.csv') # This will load the component quantity sample into the model # from the specified file. + """ self.log.div() self.log.msg('Loading asset components sample...') @@ -213,21 +217,50 @@ def load_cmp_sample(self, filepath: str) -> None: return_units=True, log=self._asmnt.log, ) - - sample.columns.names = ['cmp', 'loc', 'dir', 'uid'] + assert isinstance(sample, pd.DataFrame) + assert isinstance(units, pd.Series) + + # Check if a `uid` level was passed + num_levels = len(sample.columns.names) + num_levels_without_uid = 3 + num_levels_with_uid = num_levels_without_uid + 1 + if num_levels == num_levels_without_uid: + # No `uid`, add one. + sample.columns.names = ['cmp', 'loc', 'dir'] + sample = base.dedupe_index(sample.T).T + elif num_levels == num_levels_with_uid: + sample.columns.names = ['cmp', 'loc', 'dir', 'uid'] + else: + msg = ( + f'Invalid component sample: Column MultiIndex ' + f'has an unexpected length: {num_levels}' + ) + raise ValueError(msg) self.cmp_sample = sample self.cmp_units = units.groupby(level=0).first() + # Add marginal parameters with Blocks information (later calls + # rely on that attribute being defined) + # Obviously we can't trace back the distributions and their + # parameters, those columns are left undefined. + cmp_marginal_params = pd.DataFrame( + self.cmp_sample.columns.to_list(), columns=self.cmp_sample.columns.names + ).astype(str) + cmp_marginal_params['Blocks'] = 1 + cmp_marginal_params = cmp_marginal_params.set_index( + ['cmp', 'loc', 'dir', 'uid'] + ) + self.cmp_marginal_params = cmp_marginal_params + self.log.msg( 'Asset components sample successfully loaded.', prepend_timestamp=False ) def load_cmp_model(self, data_source: str | dict[str, pd.DataFrame]) -> None: """ - Loads the model describing component quantities in an asset - from specified data sources. + Load the asset model from a specified data source. This function is responsible for loading data related to the component model of an asset. It supports loading from multiple @@ -238,7 +271,7 @@ def load_cmp_model(self, data_source: str | dict[str, pd.DataFrame]) -> None: Parameters ---------- - data_source : str or dict + data_source: str or dict The source from where to load the component model data. If it's a string, it should be the prefix for three files: one for marginal distributions (`_marginals.csv`), @@ -273,14 +306,6 @@ def load_cmp_model(self, data_source: str | dict[str, pd.DataFrame]) -> None: >>> model.load_cmp_model(data_dict) """ - - def get_attribute(attribute_str, dtype=float, default=np.nan): - # pylint: disable=missing-return-doc - # pylint: disable=missing-return-type-doc - if pd.isnull(attribute_str): - return default - return dtype(attribute_str) - self.log.div() self.log.msg('Loading component model...') @@ -290,7 +315,7 @@ def get_attribute(attribute_str, dtype=float, default=np.nan): # prepare the marginal data source variable to load the data if isinstance(data_source, dict): - marginal_data_source = data_source['marginals'] + marginal_data_source: pd.DataFrame | str = data_source['marginals'] else: marginal_data_source = data_source + '_marginals.csv' @@ -302,13 +327,15 @@ def get_attribute(attribute_str, dtype=float, default=np.nan): return_units=True, log=self._asmnt.log, ) + assert isinstance(marginal_params, pd.DataFrame) + assert isinstance(units, pd.Series) # group units by cmp id to avoid redundant entries self.cmp_units = units.copy().groupby(level=0).first() marginal_params = pd.concat([marginal_params, units], axis=1) - cmp_marginal_param_dct = { + cmp_marginal_param_dct: dict[str, list[Any]] = { 'Family': [], 'Theta_0': [], 'Theta_1': [], @@ -320,19 +347,17 @@ def get_attribute(attribute_str, dtype=float, default=np.nan): } index_list = [] for row in marginal_params.itertuples(): - locs = self._get_locations(row.Location) - dirs = self._get_directions(row.Direction) + locs = self._get_locations(str(row.Location)) + dirs = self._get_directions(str(row.Direction)) indices = list(product((row.Index,), locs, dirs)) num_vals = len(indices) for col, cmp_marginal_param in cmp_marginal_param_dct.items(): if col == 'Blocks': cmp_marginal_param.extend( [ - get_attribute( - getattr(row, 'Blocks', np.nan), - dtype=int, - default=1.0, - ) + int(row.Blocks) # type: ignore + if ('Blocks' in dir(row) and not pd.isna(row.Blocks)) + else 1, ] * num_vals ) @@ -342,7 +367,7 @@ def get_attribute(attribute_str, dtype=float, default=np.nan): cmp_marginal_param.extend([getattr(row, col, np.nan)] * num_vals) else: cmp_marginal_param.extend( - [get_attribute(getattr(row, col, np.nan))] * num_vals + [str(getattr(row, col, np.nan))] * num_vals ) index_list.extend(indices) index = pd.MultiIndex.from_tuples(index_list, names=['cmp', 'loc', 'dir']) @@ -366,25 +391,27 @@ def get_attribute(attribute_str, dtype=float, default=np.nan): cmp_marginal_params = pd.concat(cmp_marginal_param_series, axis=1) - assert not cmp_marginal_params['Theta_0'].isnull().values.any() + assert not ( + cmp_marginal_params['Theta_0'].isna().to_numpy().any() # type: ignore + ) - cmp_marginal_params.dropna(axis=1, how='all', inplace=True) + cmp_marginal_params = cmp_marginal_params.dropna(axis=1, how='all') self.log.msg( - "Model parameters successfully parsed. " - f"{cmp_marginal_params.shape[0]} performance groups identified", + 'Model parameters successfully parsed. ' + f'{cmp_marginal_params.shape[0]} performance groups identified', prepend_timestamp=False, ) # Now we can take care of converting the values to base units self.log.msg( - "Converting model parameters to internal units...", + 'Converting model parameters to internal units...', prepend_timestamp=False, ) # ensure that the index has unique entries by introducing an # internal component uid - base.dedupe_index(cmp_marginal_params) + cmp_marginal_params = base.dedupe_index(cmp_marginal_params) cmp_marginal_params = self._convert_marginal_params( cmp_marginal_params, cmp_marginal_params['Units'] @@ -393,26 +420,19 @@ def get_attribute(attribute_str, dtype=float, default=np.nan): self.cmp_marginal_params = cmp_marginal_params.drop('Units', axis=1) self.log.msg( - "Model parameters successfully loaded.", prepend_timestamp=False + 'Model parameters successfully loaded.', prepend_timestamp=False ) self.log.msg( - "\nComponent model marginal distributions:\n" + str(cmp_marginal_params), + '\nComponent model marginal distributions:\n' + str(cmp_marginal_params), prepend_timestamp=False, ) # the empirical data and correlation files can be added later, if needed - def list_unique_component_ids( - self, as_set: bool = False - ) -> list[str] | set[str]: + def list_unique_component_ids(self) -> list[str]: """ - Returns unique component IDs. - - Parameters - ---------- - as_set: bool - Whether to cast the list into a set. + Obtain unique component IDs. Returns ------- @@ -420,13 +440,13 @@ def list_unique_component_ids( Unique components in the asset model. """ - cmp_list = self.cmp_marginal_params.index.unique(level=0).to_list() - if as_set: - return set(cmp_list) - return cmp_list + assert self.cmp_marginal_params is not None + return self.cmp_marginal_params.index.unique(level=0).to_list() def generate_cmp_sample(self, sample_size: int | None = None) -> None: """ + Generate a component sample. + Generates a sample of component quantity realizations based on predefined model parameters and optionally specified sample size. If no sample size is provided, the function attempts to @@ -445,76 +465,86 @@ def generate_cmp_sample(self, sample_size: int | None = None) -> None: If the model parameters are not loaded before sample generation, or if neither sample size is specified nor can be determined from the demand model. - """ + """ if self.cmp_marginal_params is None: - raise ValueError( - 'Model parameters have not been specified. Load' + msg = ( + 'Model parameters have not been specified. Load ' 'parameters from a file before generating a ' 'sample.' ) + raise ValueError(msg) self.log.div() self.log.msg('Generating sample from component quantity variables...') if sample_size is None: if self._asmnt.demand.sample is None: - raise ValueError( + msg = ( 'Sample size was not specified, ' 'and it cannot be determined from ' 'the demand model.' ) + raise ValueError(msg) sample_size = self._asmnt.demand.sample.shape[0] self._create_cmp_RVs() + assert self._cmp_RVs is not None + assert self._asmnt.options.sampling_method is not None self._cmp_RVs.generate_sample( sample_size=sample_size, method=self._asmnt.options.sampling_method ) cmp_sample = pd.DataFrame(self._cmp_RVs.RV_sample) - cmp_sample.sort_index(axis=0, inplace=True) - cmp_sample.sort_index(axis=1, inplace=True) - cmp_sample = base.convert_to_MultiIndex(cmp_sample, axis=1)['CMP'] + cmp_sample = cmp_sample.sort_index(axis=0) + cmp_sample = cmp_sample.sort_index(axis=1) + cmp_sample_mi = base.convert_to_MultiIndex(cmp_sample, axis=1)['CMP'] + assert isinstance(cmp_sample_mi, pd.DataFrame) + cmp_sample = cmp_sample_mi cmp_sample.columns.names = ['cmp', 'loc', 'dir', 'uid'] self.cmp_sample = cmp_sample self.log.msg( - f"\nSuccessfully generated {sample_size} realizations.", + f'\nSuccessfully generated {sample_size} realizations.', prepend_timestamp=False, ) - def _create_cmp_RVs(self) -> None: - """ - Defines the RVs used for sampling component quantities. - """ - + def _create_cmp_RVs(self) -> None: # noqa: N802 + """Define the RVs used for sampling component quantities.""" # initialize the registry - RV_reg = uq.RandomVariableRegistry(self._asmnt.options.rng) + rv_reg = uq.RandomVariableRegistry(self._asmnt.options.rng) # add a random variable for each component quantity variable + assert self.cmp_marginal_params is not None for rv_params in self.cmp_marginal_params.itertuples(): cmp = rv_params.Index # create a random variable and add it to the registry - family = getattr(rv_params, "Family", 'deterministic') - RV_reg.add_RV( + family = getattr(rv_params, 'Family', 'deterministic') + rv_reg.add_RV( uq.rv_class_map(family)( - name=f'CMP-{cmp[0]}-{cmp[1]}-{cmp[2]}-{cmp[3]}', - theta=[ - getattr(rv_params, f"Theta_{t_i}", np.nan) - for t_i in range(3) - ], - truncation_limits=[ - getattr(rv_params, f"Truncate{side}", np.nan) - for side in ("Lower", "Upper") - ], + name=f'CMP-{cmp[0]}-{cmp[1]}-{cmp[2]}-{cmp[3]}', # type: ignore + theta=np.array( + [ + value + for t_i in range(3) + if (value := getattr(rv_params, f'Theta_{t_i}', None)) + is not None + ] + ), + truncation_limits=np.array( + [ + getattr(rv_params, f'Truncate{side}', np.nan) + for side in ('Lower', 'Upper') + ] + ), ) ) self.log.msg( - f"\n{self.cmp_marginal_params.shape[0]} random variables created.", + f'\n{self.cmp_marginal_params.shape[0]} random variables created.', prepend_timestamp=False, ) - self._cmp_RVs = RV_reg + self._cmp_RVs = rv_reg diff --git a/pelicun/model/damage_model.py b/pelicun/model/damage_model.py index cb2bc8ed7..8977df1d2 100644 --- a/pelicun/model/damage_model.py +++ b/pelicun/model/damage_model.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -38,51 +37,56 @@ # Adam Zsarnóczay # John Vouvakis Manousakis -""" -This file defines the DamageModel object and its methods. -.. rubric:: Contents - -.. autosummary:: - - DamageModel - -""" +"""DamageModel object and methods.""" from __future__ import annotations + +from collections import defaultdict +from functools import partial +from pathlib import Path from typing import TYPE_CHECKING + import numpy as np import pandas as pd + +from pelicun import base, file_io, uq +from pelicun.model.demand_model import ( + _assemble_required_demand_data, + _get_required_demand_type, + _verify_edps_available, +) from pelicun.model.pelicun_model import PelicunModel -from pelicun.model.demand_model import _get_required_demand_type -from pelicun.model.demand_model import _assemble_required_demand_data -from pelicun.model.demand_model import _verify_edps_available -from pelicun import base -from pelicun import uq -from pelicun import file_io if TYPE_CHECKING: - from pelicun.assessment import Assessment + from pelicun.assessment import AssessmentBase + from pelicun.uq import RandomVariableRegistry idx = base.idx class DamageModel(PelicunModel): - """ - Manages damage information used in assessments. - - """ + """Manages damage information used in assessments.""" __slots__ = ['ds_model', 'missing_components'] - def __init__(self, assessment: Assessment): + def __init__(self, assessment: AssessmentBase) -> None: + """ + Initialize a Damage model. + + Parameters + ---------- + assessment: AssessmentBase + The parent assessment object. + + """ super().__init__(assessment) self.ds_model: DamageModel_DS = DamageModel_DS(assessment) self.missing_components: list[str] = [] @property - def _damage_models(self): + def _damage_models(self) -> tuple[DamageModel_DS]: """ Points to the damage model objects included in DamageModel. @@ -95,29 +99,26 @@ def _damage_models(self): return (self.ds_model,) def load_damage_model( - self, data_paths: list[str | pd.DataFrame], warn_missing: bool = False + self, data_paths: list[str | pd.DataFrame], *, warn_missing: bool = False ) -> None: - """ - - - """ - self.log.warn( + """.""" + self.log.warning( '`load_damage_model` is deprecated and will be ' 'dropped in future versions of pelicun. ' 'Please use `load_model_parameters` instead, ' - 'like so: \n`cmp_set = {your_assessment_obj}.' + 'like so: \n`cmp_set = set({your_assessment_obj}.' 'asset.' - 'list_unique_component_ids(as_set=True)`, ' + 'list_unique_component_ids())`, ' 'and then \n`{your_assessment_obj}.damage.' 'load_model_parameters(data_paths, cmp_set)`.' ) - cmp_set = self._asmnt.asset.list_unique_component_ids(as_set=True) - self.load_model_parameters(data_paths, cmp_set, warn_missing) + cmp_set = set(self._asmnt.asset.list_unique_component_ids()) + self.load_model_parameters(data_paths, cmp_set, warn_missing=warn_missing) @property def sample(self) -> pd.DataFrame: """ - + . Returns ------- @@ -125,18 +126,20 @@ def sample(self) -> pd.DataFrame: The damage sample of the `ds_model`. """ - self.log.warn( + self.log.warning( '`{damage model}.sample` is deprecated and will be ' 'dropped in future versions of pelicun. ' 'Please use `{damage model}.ds_model.sample` instead. ' 'Now returning `{damage model}.ds_model.sample`.' ) + assert self.ds_model.sample is not None return self.ds_model.sample def load_model_parameters( self, data_paths: list[str | pd.DataFrame], cmp_set: set[str], + *, warn_missing: bool = False, ) -> None: """ @@ -156,7 +159,7 @@ def load_model_parameters( Damage parameters in the input files for components outside of that set are omitted for performance. warn_missing: bool - Wether to check if there are components in the asset model + Whether to check if there are components in the asset model that do not have specified damage parameters. Should be set to True if all components in the asset model are damage state-driven, or if only a damage estimation is @@ -169,15 +172,14 @@ def load_model_parameters( specified paths. """ - self.log.div() self.log.msg('Loading damage model...', prepend_timestamp=False) # for i, path in enumerate(data_paths): if 'fragility_DB' in path: - path = path.replace('fragility_DB', 'damage_DB') - self.log.warn( + path = path.replace('fragility_DB', 'damage_DB') # noqa: PLW2901 + self.log.warning( '`fragility_DB` is deprecated and will ' 'be dropped in future versions of pelicun. ' 'Please use `damage_DB` instead.' @@ -197,10 +199,12 @@ def load_model_parameters( ) # determine if the damage model parameters are for damage # states + assert isinstance(data, pd.DataFrame) if _is_for_ds_model(data): - self.ds_model._load_model_parameters(data) + self.ds_model.load_model_parameters(data) else: - raise ValueError(f'Invalid damage model parameters: {data_path}') + msg = f'Invalid damage model parameters: {data_path}' + raise ValueError(msg) self.log.msg( 'Damage model parameters loaded successfully.', prepend_timestamp=False @@ -216,9 +220,9 @@ def load_model_parameters( for damage_model in self._damage_models: # drop unused damage parameter definitions - damage_model._drop_unused_damage_parameters(cmp_set) + damage_model.drop_unused_damage_parameters(cmp_set) # remove components with incomplete damage parameters - damage_model._remove_incomplete_components() + damage_model.remove_incomplete_components() # # convert units @@ -228,7 +232,7 @@ def load_model_parameters( 'Converting damage model parameter units.', prepend_timestamp=False ) for damage_model in self._damage_models: - damage_model._convert_damage_parameter_units() + damage_model.convert_damage_parameter_units() # # verify damage parameter availability @@ -240,7 +244,7 @@ def load_model_parameters( prepend_timestamp=False, ) missing_components = self._ensure_damage_parameter_availability( - cmp_set, warn_missing + cmp_set, warn_missing=warn_missing ) self.missing_components = missing_components @@ -254,11 +258,28 @@ def calculate( """ Calculate the damage of each component block. - """ + Parameters + ---------- + dmg_process: dict, optional + Allows simulating damage processes, where damage to some + component can alter the damage state of other components. + block_batch_size: int + Maximum number of components in each batch. + scaling_specification: dict, optional + A dictionary defining the shift in median. + Example: {'CMP-1-1': '*1.2', 'CMP-1-2': '/1.4'} + The keys are individual components that should be present + in the `capacity_sample`. The values should be strings + containing an operation followed by the value formatted as + a float. The operation can be '+' for addition, '-' for + subtraction, '*' for multiplication, and '/' for division. + """ self.log.div() self.log.msg('Calculating damages...') + assert self._asmnt.asset.cmp_sample is not None + assert self._asmnt.asset.cmp_marginal_params is not None self.log.msg( f'Number of Performance Groups in Asset Model:' f' {self._asmnt.asset.cmp_sample.shape[1]}', @@ -284,7 +305,8 @@ def calculate( ) # obtain damage states for applicable components - self.ds_model._obtain_ds_sample( + assert self._asmnt.demand.sample is not None + self.ds_model.obtain_ds_sample( demand_sample=self._asmnt.demand.sample, component_blocks=component_blocks, block_batch_size=block_batch_size, @@ -295,20 +317,20 @@ def calculate( # Apply the prescribed damage process, if any if dmg_process is not None: - self.log.msg("Applying damage processes.") + self.log.msg('Applying damage processes.') # Sort the damage processes tasks dmg_process = {key: dmg_process[key] for key in sorted(dmg_process)} # Perform damage tasks in the sorted order for task in dmg_process.items(): - self.ds_model._perform_dmg_task(task) + self.ds_model.perform_dmg_task(task) self.log.msg( - "Damage processes successfully applied.", prepend_timestamp=False + 'Damage processes successfully applied.', prepend_timestamp=False ) - qnt_sample = self.ds_model._prepare_dmg_quantities( + qnt_sample = self.ds_model.prepare_dmg_quantities( self._asmnt.asset.cmp_sample, self._asmnt.asset.cmp_marginal_params, dropzero=False, @@ -316,16 +338,18 @@ def calculate( # If requested, extend the quantity table with all possible DSs if self._asmnt.options.list_all_ds: - qnt_sample = self.ds_model._complete_ds_cols(qnt_sample) + qnt_sample = self.ds_model.complete_ds_cols(qnt_sample) self.ds_model.sample = qnt_sample self.log.msg('Damage calculation completed.', prepend_timestamp=False) def save_sample( - self, filepath: str | None = None, save_units: bool = False + self, filepath: str | None = None, *, save_units: bool = False ) -> pd.DataFrame | tuple[pd.DataFrame, pd.Series] | None: """ + Save or return the damage sample data. + Saves the damage sample data to a CSV file or returns it directly with an option to include units. @@ -337,11 +361,11 @@ def save_sample( Parameters ---------- - filepath : str, optional + filepath: str, optional The path to the file where the damage sample should be saved. If not provided, the sample is not saved to disk but returned. - save_units : bool, default: False + save_units: bool, default: False Indicates whether to include a row with unit information in the returned DataFrame. This parameter is ignored if a file path is provided. @@ -355,20 +379,25 @@ def save_sample( - DataFrame containing the damage sample. - Optionally, a Series containing the units for each column if `save_units` is True. + """ self.log.div() self.log.msg('Saving damage sample...') + if self.ds_model.sample is None: + return None + cmp_units = self._asmnt.asset.cmp_units qnt_units = pd.Series( index=self.ds_model.sample.columns, name='Units', dtype='object' ) + assert cmp_units is not None for cmp in cmp_units.index: qnt_units.loc[cmp] = cmp_units.loc[cmp] res = file_io.save_to_csv( self.ds_model.sample, - filepath, + Path(filepath) if filepath is not None else None, units=qnt_units, unit_conversion_factors=self._asmnt.unit_conversion_factors, use_simpleindex=(filepath is not None), @@ -382,26 +411,29 @@ def save_sample( return None # else: - units = res.loc["Units"] - res.drop("Units", inplace=True) + assert isinstance(res, pd.DataFrame) + units = res.loc['Units'] + assert isinstance(units, pd.Series) + res = res.drop('Units') res.index = res.index.astype('int64') + res = res.astype(float) + assert isinstance(res, pd.DataFrame) if save_units: - return res.astype(float), units + return res, units - return res.astype(float) + return res def load_sample(self, filepath: str) -> None: - """ - Load damage state sample data. - - """ + """Load damage state sample data.""" self.log.div() self.log.msg('Loading damage sample...') - self.ds_model.sample = file_io.load_data( + data = file_io.load_data( filepath, self._asmnt.unit_conversion_factors, log=self._asmnt.log ) + assert isinstance(data, pd.DataFrame) + self.ds_model.sample = data # set the names of the columns self.ds_model.sample.columns.names = ['cmp', 'loc', 'dir', 'uid', 'ds'] @@ -409,17 +441,17 @@ def load_sample(self, filepath: str) -> None: self.log.msg('Damage sample successfully loaded.', prepend_timestamp=False) def _ensure_damage_parameter_availability( - self, cmp_list: list[str], warn_missing: bool + self, cmp_set: set[str], *, warn_missing: bool ) -> list[str]: """ - Makes sure that all components have damage parameters. + Make sure that all components have damage parameters. Parameters ---------- - cmp_list: list + cmp_set: list List of component IDs in the asset model. warn_missing: bool - Wether to issue a warning if missing components are found. + Whether to issue a warning if missing components are found. Returns ------- @@ -427,28 +459,32 @@ def _ensure_damage_parameter_availability( List of component IDs with missing damage parameters. """ - available_components = self._get_component_id_set() missing_components = [ component - for component in cmp_list + for component in cmp_set if component not in available_components ] if missing_components and warn_missing: - self.log.warn( - f"The damage model does not provide " - f"damage information for the following component(s) " - f"in the asset model: {missing_components}." + self.log.warning( + f'The damage model does not provide ' + f'damage information for the following component(s) ' + f'in the asset model: {missing_components}.' ) return missing_components def _get_component_id_set(self) -> set[str]: """ - Get a set of components for which damage parameters are - available. + Get a set of components with available damage parameters. + + Returns + ------- + set + Set of components with available damage parameters. + """ cmp_list = [] if self.ds_model.damage_params is not None: @@ -457,22 +493,30 @@ def _get_component_id_set(self) -> set[str]: class DamageModel_Base(PelicunModel): - """ - Base class for damage models - - """ + """Base class for damage models.""" __slots__ = ['damage_params', 'sample'] - def __init__(self, assessment: Assessment): + def __init__(self, assessment: AssessmentBase) -> None: + """ + Initialize the object. + + Parameters + ---------- + assessment: AssessmentBase + Parent assessment object. + + """ super().__init__(assessment) - self.damage_params = None - self.sample = None + self.damage_params: pd.DataFrame | None = None + self.sample: pd.DataFrame | None = None - def _load_model_parameters(self, data: pd.DataFrame) -> None: + def load_model_parameters(self, data: pd.DataFrame) -> None: """ - Load model parameters from a DataFrame, extending those + Load model parameters from a DataFrame. + + Loads model parameters from a DataFrame, extending those already available. Parameters already defined take precedence, i.e. redefinitions of parameters are ignored. @@ -482,35 +526,35 @@ def _load_model_parameters(self, data: pd.DataFrame) -> None: Data with damage model information. """ - if self.damage_params is not None: data = pd.concat((self.damage_params, data), axis=0) # drop redefinitions of components data = data.groupby(data.index).first() - # TODO: load defaults for Demand-Offset and Demand-Directional + # TODO(AZ): load defaults for Demand-Offset and Demand-Directional self.damage_params = data - def _convert_damage_parameter_units(self) -> None: - """ - Converts previously loaded damage parameters to base units. - - """ + def convert_damage_parameter_units(self) -> None: + """Convert previously loaded damage parameters to base units.""" if self.damage_params is None: return - units = self.damage_params[('Demand', 'Unit')] - self.damage_params.drop(('Demand', 'Unit'), axis=1, inplace=True) - for LS_i in self.damage_params.columns.unique(level=0): - if LS_i.startswith('LS'): - self.damage_params.loc[:, LS_i] = self._convert_marginal_params( - self.damage_params.loc[:, LS_i].copy(), units - ).values - - def _remove_incomplete_components(self) -> None: + units = self.damage_params['Demand', 'Unit'] + self.damage_params = self.damage_params.drop(('Demand', 'Unit'), axis=1) + for ls_i in self.damage_params.columns.unique(level=0): + if ls_i.startswith('LS'): + params = self.damage_params.loc[:, ls_i].copy() + assert isinstance(params, pd.DataFrame) + self.damage_params.loc[:, ls_i] = self._convert_marginal_params( + params, units + ).to_numpy() + + def remove_incomplete_components(self) -> None: """ + Remove components with incompelte damage parameter info. + Removes components that have incomplete damage model definitions from the damage model parameters. @@ -522,13 +566,15 @@ def _remove_incomplete_components(self) -> None: return cmp_incomplete_idx = self.damage_params.loc[ - self.damage_params[('Incomplete', '')] == 1 + self.damage_params['Incomplete', ''] == 1 ].index - self.damage_params.drop(cmp_incomplete_idx, inplace=True) + self.damage_params = self.damage_params.drop(cmp_incomplete_idx) - def _drop_unused_damage_parameters(self, cmp_set: set[str]) -> None: + def drop_unused_damage_parameters(self, cmp_set: set[str]) -> None: """ + Remove info for non existent components. + Removes damage parameter definitions for component IDs not present in the given list. @@ -537,6 +583,7 @@ def _drop_unused_damage_parameters(self, cmp_set: set[str]) -> None: cmp_set: set Set of component IDs to be preserved in the damage parameters. + """ if self.damage_params is None: return @@ -550,8 +597,7 @@ def _get_pg_batches( missing_components: list[str], ) -> pd.DataFrame: """ - Group performance groups into batches for efficient damage - assessment. + Group performance groups into batches for efficiency. The method takes as input the block_batch_size, which specifies the maximum number of blocks per batch. The method @@ -577,9 +623,8 @@ def _get_pg_batches( Parameters ---------- - component_blocks: pd.DataFrame - DataFrame containing a singe column, `Blocks`, which lists + DataFrame containing a single column, `Blocks`, which lists the number of blocks for each (`cmp`-`loc`-`dir`-`uid`). block_batch_size: int Maximum number of components in each batch. @@ -599,15 +644,15 @@ def _get_pg_batches( block batch size. """ - # A warning has already been issued for components with # missing damage parameters (in # `DamageModel._ensure_damage_parameter_availability`). - component_blocks.drop(pd.Index(missing_components), inplace=True) + component_blocks = component_blocks.drop(pd.Index(missing_components)) # It is safe to simply disregard components that are not # present in the `damage_params` of *this* model, and let them # be handled by another damage model. + assert self.damage_params is not None available_components = self.damage_params.index.unique().to_list() component_blocks = component_blocks.loc[ pd.IndexSlice[available_components, :, :, :], : @@ -617,11 +662,11 @@ def _get_pg_batches( component_blocks = component_blocks.groupby( ['loc', 'dir', 'cmp', 'uid'] ).sum() - component_blocks.sort_index(axis=0, inplace=True) + component_blocks = component_blocks.sort_index(axis=0) # Calculate cumulative sum of blocks component_blocks['CBlocks'] = np.cumsum( - component_blocks['Blocks'].values.astype(int) + component_blocks['Blocks'].to_numpy().astype(int) ) component_blocks['Batch'] = 0 @@ -640,7 +685,7 @@ def _get_pg_batches( ) if np.sum(batch_mask) < 1: - batch_mask = np.full(batch_mask.shape, False) + batch_mask = np.full(batch_mask.shape, fill_value=False) batch_mask[np.where(component_blocks['CBlocks'] > 0)[0][0]] = True component_blocks.loc[batch_mask, 'Batch'] = batch_i @@ -665,28 +710,32 @@ def _get_pg_batches( .loc[:, 'Blocks'] .to_frame() ) - component_blocks = component_blocks.sort_index( + return component_blocks.sort_index( level=['Batch', 'cmp', 'loc', 'dir', 'uid'] ) - return component_blocks class DamageModel_DS(DamageModel_Base): - """ - Damage model for components that have discrete Damage States (DS). - - """ + """Damage model for components that have discrete Damage States (DS).""" __slots__ = ['ds_sample'] - def __init__(self, assessment: Assessment): + def __init__(self, assessment: AssessmentBase) -> None: + """ + Initialize the object. + + Parameters + ---------- + assessment: AssessmentBase + Parent assessment object. + + """ super().__init__(assessment) - self.ds_sample = None + self.ds_sample: pd.DataFrame | None = None def probabilities(self) -> pd.DataFrame: """ - Returns the probability of each observed damage state in the - sample. + Return the probability of each observed damage state. Returns ------- @@ -696,6 +745,7 @@ def probabilities(self) -> pd.DataFrame: """ sample = self.ds_sample + assert sample is not None probabilities = {} @@ -708,7 +758,7 @@ def probabilities(self) -> pd.DataFrame: probabilities[col] = np.nan else: vcounts = values.value_counts() / len(values) - probabilities[col] = vcounts + probabilities[col] = vcounts # type: ignore return ( pd.DataFrame(probabilities) @@ -719,7 +769,7 @@ def probabilities(self) -> pd.DataFrame: .sort_index(axis=0) ) - def _obtain_ds_sample( + def obtain_ds_sample( self, demand_sample: pd.DataFrame, component_blocks: pd.DataFrame, @@ -728,11 +778,7 @@ def _obtain_ds_sample( missing_components: list[str], nondirectional_multipliers: dict[str, float], ) -> None: - """ - Obtain the damage state of each performance group in the - model. - """ - + """Obtain the damage state of each performance group.""" # Break up damage calculation and perform it by performance group. # Compared to the simultaneous calculation of all PGs, this approach # reduces demands on memory and increases the load on CPU. This leads @@ -753,19 +799,18 @@ def _obtain_ds_sample( ) self.log.msg( - f"{len(batches)} batches of Performance Groups prepared " - "for damage assessment", + f'{len(batches)} batches of Performance Groups prepared ' + 'for damage assessment', prepend_timestamp=False, ) # for PG_i in self._asmnt.asset.cmp_sample.columns: ds_samples = [] - for PGB_i in batches: - - performance_group = component_blocks.loc[PGB_i] + for pgb_i in batches: + performance_group = component_blocks.loc[pgb_i] self.log.msg( - f"Calculating damage states for PG batch {PGB_i} with " + f"Calculating damage states for PG batch {pgb_i} with " f"{int(performance_group['Blocks'].sum())} blocks" ) @@ -784,6 +829,7 @@ def _obtain_ds_sample( prepend_timestamp=True, ) demand_offset = self._asmnt.options.demand_offset + assert self.damage_params is not None required_edps = _get_required_demand_type( self.damage_params, performance_group, demand_offset ) @@ -818,12 +864,14 @@ def _obtain_ds_sample( self.ds_sample = pd.concat(ds_samples, axis=1) - self.log.msg("Damage state calculation successful.", prepend_timestamp=False) + self.log.msg('Damage state calculation successful.', prepend_timestamp=False) - def _handle_operation( + def _handle_operation( # noqa: PLR6301 self, initial_value: float, operation: str, other_value: float ) -> float: """ + Handle a capacity adjustment operation. + This method is used in `_create_dmg_RVs` to apply capacity adjustment operations whenever required. It is defined as a safer alternative to directly using `eval`. @@ -856,26 +904,64 @@ def _handle_operation( return initial_value * other_value if operation == '/': return initial_value / other_value - raise ValueError(f'Invalid operation: `{operation}`') + msg = f'Invalid operation: `{operation}`' + raise ValueError(msg) + + def _handle_operation_list( + self, initial_value: float, operations: list[tuple[str, float]] + ) -> np.ndarray: + """ + Apply one or more operations to an initial value and return the results. + + Parameters. + ---------- + initial_value : float + The initial value to which the operations will be applied. + operations : list of tuple + A list of operations where each operation is represented as a tuple. + The first element of the tuple is a string representing the operation + type, and the second element is a float representing the value to be + used in the operation. + + Returns + ------- + np.ndarray + An array of results after applying each operation to the initial value. + """ + if len(operations) == 1: + return np.array( + [ + self._handle_operation( + initial_value, operations[0][0], operations[0][1] + ) + ] + ) + new_values = [ + self._handle_operation(initial_value, operation[0], operation[1]) + for operation in operations + ] + return np.array(new_values) def _generate_dmg_sample( self, sample_size: int, - PGB: pd.DataFrame, + pgb: pd.DataFrame, scaling_specification: dict | None = None, ) -> tuple[pd.DataFrame, pd.DataFrame]: """ - This method generates a damage sample by creating random - variables (RVs) for capacities and limit-state-damage-states - (lsds), and then sampling from these RVs. The sample size and - performance group batches (PGB) are specified as inputs. The - method returns the capacity sample and the lsds sample. + Generate the damage sample. + + Generates a damage sample by creating random variables (RVs) + for capacities and limit-state-damage-states (lsds), and then + sampling from these RVs. The sample size and performance group + batches (PGB) are specified as inputs. The method returns the + capacity sample and the lsds sample. Parameters ---------- - sample_size : int + sample_size: int The number of realizations to generate. - PGB : DataFrame + pgb: DataFrame A DataFrame that groups performance groups into batches for efficient damage assessment. scaling_specification: dict, optional @@ -889,9 +975,9 @@ def _generate_dmg_sample( Returns ------- - capacity_sample : DataFrame + capacity_sample: DataFrame A DataFrame that represents the capacity sample. - lsds_sample : DataFrame + lsds_sample: DataFrame A DataFrame that represents the . Raises @@ -902,53 +988,61 @@ def _generate_dmg_sample( """ # Check if damage model parameters have been specified if self.damage_params is None: - raise ValueError( + msg = ( 'Damage model parameters have not been specified. ' 'Load parameters from the default damage model ' 'databases or provide your own damage model ' 'definitions before generating a sample.' ) + raise ValueError(msg) # Create capacity and LSD RVs for each performance group - capacity_RVs, lsds_RVs = self._create_dmg_RVs(PGB, scaling_specification) + capacity_rvs, lsds_rvs = self._create_dmg_RVs(pgb, scaling_specification) if self._asmnt.log.verbose: self.log.msg('Sampling capacities...', prepend_timestamp=True) # Generate samples for capacity RVs - capacity_RVs.generate_sample( + assert self._asmnt.options.sampling_method is not None + capacity_rvs.generate_sample( sample_size=sample_size, method=self._asmnt.options.sampling_method ) # Generate samples for LSD RVs - lsds_RVs.generate_sample( + lsds_rvs.generate_sample( sample_size=sample_size, method=self._asmnt.options.sampling_method ) if self._asmnt.log.verbose: - self.log.msg("Raw samples are available", prepend_timestamp=True) + self.log.msg('Raw samples are available', prepend_timestamp=True) # get the capacity and lsds samples capacity_sample = ( - pd.DataFrame(capacity_RVs.RV_sample) + pd.DataFrame(capacity_rvs.RV_sample) .sort_index(axis=0) .sort_index(axis=1) ) - capacity_sample = base.convert_to_MultiIndex(capacity_sample, axis=1)['FRG'] + capacity_sample_mi = base.convert_to_MultiIndex(capacity_sample, axis=1)[ + 'FRG' + ] + assert isinstance(capacity_sample_mi, pd.DataFrame) + capacity_sample = capacity_sample_mi capacity_sample.columns.names = ['cmp', 'loc', 'dir', 'uid', 'block', 'ls'] lsds_sample = ( - pd.DataFrame(lsds_RVs.RV_sample) + pd.DataFrame(lsds_rvs.RV_sample) .sort_index(axis=0) .sort_index(axis=1) .astype(int) ) - lsds_sample = base.convert_to_MultiIndex(lsds_sample, axis=1)['LSDS'] + lsds_sample_mi = base.convert_to_MultiIndex(lsds_sample, axis=1)['LSDS'] + assert isinstance(lsds_sample_mi, pd.DataFrame) + lsds_sample = lsds_sample_mi lsds_sample.columns.names = ['cmp', 'loc', 'dir', 'uid', 'block', 'ls'] if self._asmnt.log.verbose: self.log.msg( - f"Successfully generated {sample_size} realizations.", + f'Successfully generated {sample_size} realizations.', prepend_timestamp=True, ) @@ -962,7 +1056,7 @@ def _evaluate_damage_state( lsds_sample: pd.DataFrame, ) -> pd.DataFrame: """ - Use the demand and LS capacity sample to evaluate damage states + Use the demand and LS capacity sample to evaluate damage states. Parameters ---------- @@ -982,8 +1076,8 @@ def _evaluate_damage_state( DataFrame Assigns a Damage State to each component block in the asset model. - """ + """ if self._asmnt.log.verbose: self.log.msg('Evaluating damage states...', prepend_timestamp=True) @@ -999,34 +1093,36 @@ def _evaluate_damage_state( # For each demand type in the demand dictionary for demand_name, demand_vals in demand_dict.items(): # Get the list of PGs assigned to this demand type - PG_list = required_edps[demand_name] + pg_list = required_edps[demand_name] # Create a list of columns for the demand data # corresponding to each PG in the PG_list - PG_cols = pd.concat( - [dmg_eval.loc[:1, PG_i] for PG_i in PG_list], axis=1, keys=PG_list + pg_cols = pd.concat( + [dmg_eval.loc[:1, PG_i] for PG_i in pg_list], # type: ignore + axis=1, + keys=pg_list, ).columns - PG_cols.names = ['cmp', 'loc', 'dir', 'uid', 'block', 'ls'] + pg_cols.names = ['cmp', 'loc', 'dir', 'uid', 'block', 'ls'] # Create a DataFrame with demand values repeated for the # number of PGs and assign the columns as PG_cols demand_df.append( pd.concat( - [pd.Series(demand_vals)] * len(PG_cols), axis=1, keys=PG_cols + [pd.Series(demand_vals)] * len(pg_cols), axis=1, keys=pg_cols ) ) # Concatenate all demand DataFrames into a single DataFrame - demand_df = pd.concat(demand_df, axis=1) + demand_df_concat = pd.concat(demand_df, axis=1) # Sort the columns of the demand DataFrame - demand_df.sort_index(axis=1, inplace=True) + demand_df_concat = demand_df_concat.sort_index(axis=1) # Evaluate the damage exceedance by subtracting demand from # capacity and checking if the result is less than zero - dmg_eval = (capacity_sample - demand_df) < 0 + dmg_eval = (capacity_sample - demand_df_concat) < 0 # Remove any columns with NaN values from the damage # exceedance DataFrame - dmg_eval.dropna(axis=1, inplace=True) + dmg_eval = dmg_eval.dropna(axis=1) # initialize the DataFrames that store the damage states and # quantities @@ -1041,15 +1137,18 @@ def _evaluate_damage_state( ls_list = dmg_eval.columns.get_level_values(5).unique() # for each consecutive limit state... - for LS_id in ls_list: + for ls_id in ls_list: # get all cmp - loc - dir - block where this limit state occurs - dmg_e_ls = dmg_eval.loc[:, idx[:, :, :, :, :, LS_id]].dropna(axis=1) + dmg_e_ls = dmg_eval.loc[ + :, # type: ignore + idx[:, :, :, :, :, ls_id], + ].dropna(axis=1) # Get the damage states corresponding to this limit state in each # block # Note that limit states with a set of mutually exclusive damage # states options have their damage state picked here. - lsds = lsds_sample.loc[:, dmg_e_ls.columns] + lsds = lsds_sample.loc[:, dmg_e_ls.columns] # type: ignore # Drop the limit state level from the columns to make the damage # exceedance DataFrame compatible with the other DataFrames in the @@ -1067,16 +1166,16 @@ def _evaluate_damage_state( # those cells in the result matrix will get overwritten by higher # damage states. ds_sample.loc[:, dmg_e_ls.columns] = ds_sample.loc[ - :, dmg_e_ls.columns + :, dmg_e_ls.columns # type: ignore ].mask(dmg_e_ls, lsds) return ds_sample - def _create_dmg_RVs( - self, PGB: pd.DataFrame, scaling_specification: dict | None = None + def _create_dmg_RVs( # noqa: N802, C901 + self, pgb: pd.DataFrame, scaling_specification: dict | None = None ) -> tuple[uq.RandomVariableRegistry, uq.RandomVariableRegistry]: """ - Creates random variables required later for the damage calculation. + Create random variables for the damage calculation. The method initializes two random variable registries, capacity_RV_reg and lsds_RV_reg, and loops through each @@ -1093,17 +1192,22 @@ def _create_dmg_RVs( Parameters ---------- - PGB : DataFrame + pgb: DataFrame A DataFrame that groups performance groups into batches for efficient damage assessment. scaling_specification: dict, optional - A dictionary defining the shift in median. - Example: {'CMP-1-1': '*1.2', 'CMP-1-2': '/1.4'} - The keys are individual components that should be present - in the `capacity_sample`. The values should be strings - containing an operation followed by the value formatted as - a float. The operation can be '+' for addition, '-' for - subtraction, '*' for multiplication, and '/' for division. + A dictionary defining the shift in median. + Example: {'CMP-1-1': {'LS1':['*1.2'. '*0.8'], 'LS2':'*1.2'}, + 'CMP-1-2': {'ALL':'/1.4'}} The first level keys are individual + components that should be present in the `capacity_sample`. The + second level key is the limit state to apply the scaling to. The + values should be strings or list of strings. The strings should + contain an operation followed by the value formatted as a float. + The operation can be '+' for addition, '-' for subtraction, '*' + for multiplication, and '/' for division. If different operations + are required for different realizations, a list of strings can + be provided. When 'ALL' is used as the key, the operation will + be applied to all limit states. Returns ------- @@ -1112,18 +1216,17 @@ def _create_dmg_RVs( one for the capacity random variables and one for the LSDS assignments. - Raises - ------ - ValueError - Raises an error if the scaling specification is invalid or - if the input DataFrame does not meet the expected format. - Also, raises errors if there are any issues with the types - or formats of the data in the input DataFrame. - """ - def assign_lsds(ds_weights, ds_id, lsds_RV_reg, lsds_rv_tag): + def assign_lsds( + ds_weights: str | None, + ds_id: int, + lsds_rv_reg: RandomVariableRegistry, + lsds_rv_tag: str, + ) -> int: """ + Assign limit states to damage states. + Assigns limit states to damage states using random variables, updating the provided random variable registry. This function either creates a deterministic random @@ -1132,19 +1235,19 @@ def assign_lsds(ds_weights, ds_id, lsds_RV_reg, lsds_rv_tag): Parameters ---------- - ds_weights : str or None + ds_weights: str or None A string representing the weights of different damage states associated with a limit state, separated by '|'. If None, indicates that there is only one damage state associated with the limit state. - ds_id : int + ds_id: int The starting index for damage state IDs. This ID helps in mapping damage states to limit states. - lsds_RV_reg : RandomVariableRegistry + lsds_rv_reg: RandomVariableRegistry The registry where the newly created random variables (for mapping limit states to damage states) will be added. - lsds_rv_tag : str + lsds_rv_tag: str A unique identifier for the random variable being created, typically including identifiers for component, location, direction, and limit state. @@ -1163,39 +1266,42 @@ def assign_lsds(ds_weights, ds_id, lsds_RV_reg, lsds_rv_tag): probabilistic damage assessments. It dynamically adjusts to the number of damage states specified and applies a mapping function to correctly assign state IDs. + """ # If the limit state has a single damage state assigned # to it, we don't need random sampling - if pd.isnull(ds_weights): + if pd.isna(ds_weights): ds_id += 1 - - lsds_RV_reg.add_RV( + lsds_rv_reg.add_RV( uq.DeterministicRandomVariable( name=lsds_rv_tag, - theta=ds_id, + theta=np.array((ds_id,)), ) ) # Otherwise, we create a multinomial random variable else: + assert isinstance(ds_weights, str) # parse the DS weights - ds_weights = np.array( - ds_weights.replace(" ", "").split('|'), dtype=float + ds_weights_np = np.array( + ds_weights.replace(' ', '').split('|'), dtype=float ) - def map_ds(values, offset=int(ds_id + 1)): + def map_ds(values: np.ndarray, offset: int) -> np.ndarray: """ + Map DS indices to damage states. + Maps an array of damage state indices to their corresponding actual state IDs by applying an offset. Parameters ---------- - values : array-like + values: array-like An array of indices representing damage states. These indices are typically sequential integers starting from zero. - offset : int + offset: int The value to be added to each element in `values` to obtain the actual damage state IDs. @@ -1206,70 +1312,132 @@ def map_ds(values, offset=int(ds_id + 1)): An array where each original index from `values` has been incremented by `offset` to reflect its actual damage state ID. + """ return values + offset - lsds_RV_reg.add_RV( + lsds_rv_reg.add_RV( uq.MultinomialRandomVariable( name=lsds_rv_tag, - theta=ds_weights, - f_map=map_ds, + theta=ds_weights_np, + f_map=partial(map_ds, offset=ds_id + 1), ) ) - ds_id += len(ds_weights) + ds_id += len(ds_weights_np) return ds_id + def parse_scaling_specification(scaling_specification: dict) -> dict: # noqa: C901 + """ + Parse and validate the scaling specification, used in the '_create_dmg_RVs' method. + + Parameters + ---------- + scaling_specification: dict, optional + A dictionary defining the shift in median. + Example: {'CMP-1-1': {'LS1':['*1.2'. '*0.8'], 'LS2':'*1.2'}, + 'CMP-1-2': {'ALL':'/1.4'}} The first level keys are individual + components that should be present in the `capacity_sample`. The + second level key is the limit state to apply the scaling to. The + values should be strings or list of strings. The strings should + containing an operation followed by the value formatted as + a float. The operation can be '+' for addition, '-' for + subtraction, '*' for multiplication, and '/' for division. If + different operations are required for different realizations, a + list of strings can be provided. When 'ALL' is used as the key, + the operation will be applied to all limit states. + + Returns + ------- + dict + The parsed and validated scaling specification. + + Raises + ------ + ValueError + If the scaling specification is invalid. + TypeError + If the type of an entry is invalid. + """ + # if there are contents, ensure they are valid. + # See docstring for an example of what is expected. + parsed_scaling_specification: dict = defaultdict(dict) + # validate contents + for key, value in scaling_specification.items(): + # loop through limit states + if 'ALL' in value: + if len(value) > 1: + msg = ( + f'Invalid entry in scaling_specification: ' + f"{value}. No other entries are allowed for a component when 'ALL' is used." + ) + raise ValueError(msg) + for limit_state_id, specifics in value.items(): + if not ( + limit_state_id.startswith('LS') or limit_state_id == 'ALL' + ): + msg = ( + f'Invalid entry in scaling_specification: {limit_state_id}. ' + f"It has to start with 'LS' or be 'ALL'. " + f'See docstring of DamageModel._create_dmg_RVs.' + ) + raise ValueError(msg) + css = 'capacity adjustment specification' + if not isinstance(specifics, list): + specifics_list = [specifics] + else: + specifics_list = specifics + for spec in specifics_list: + if not isinstance(spec, str): + msg = ( + f'Invalud entry in {css}: {spec}.' + f'The specified scaling operation has to be a string.' + f'See docstring of DamageModel._create_dmg_RVs.' + ) + raise TypeError(msg) + capacity_adjustment_operation = spec[0] + number = spec[1::] + if capacity_adjustment_operation not in {'+', '-', '*', '/'}: + msg = f'Invalid operation in {css}: ' + raise ValueError(msg, f'{capacity_adjustment_operation}') + fnumber = base.float_or_None(number) + if fnumber is None: + msg = f'Invalid number in {css}: {number}' + raise ValueError(msg) + if limit_state_id not in parsed_scaling_specification[key]: + parsed_scaling_specification[key][limit_state_id] = [] + parsed_scaling_specification[key][limit_state_id].append( + (capacity_adjustment_operation, fnumber) + ) + return parsed_scaling_specification + if self._asmnt.log.verbose: self.log.msg('Generating capacity variables ...', prepend_timestamp=True) # initialize the registry - capacity_RV_reg = uq.RandomVariableRegistry(self._asmnt.options.rng) - lsds_RV_reg = uq.RandomVariableRegistry(self._asmnt.options.rng) + capacity_rv_reg = uq.RandomVariableRegistry(self._asmnt.options.rng) + lsds_rv_reg = uq.RandomVariableRegistry(self._asmnt.options.rng) # capacity adjustment: # ensure the scaling_specification is a dictionary if not scaling_specification: scaling_specification = {} else: - # if there are contents, ensure they are valid. - # See docstring for an example of what is expected. - parsed_scaling_specification = {} - # validate contents - for key, value in scaling_specification.items(): - css = 'capacity adjustment specification' - if not isinstance(value, str): - raise ValueError( - f'Invalid entry in {css}: {value}. It has to be a string. ' - f'See docstring of DamageModel._create_dmg_RVs.' - ) - capacity_adjustment_operation = value[0] - number = value[1::] - if capacity_adjustment_operation not in ('+', '-', '*', '/'): - raise ValueError( - f'Invalid operation in {css}: ' - f'{capacity_adjustment_operation}' - ) - fnumber = base.float_or_None(number) - if fnumber is None: - raise ValueError(f'Invalid number in {css}: {number}') - parsed_scaling_specification[key] = ( - capacity_adjustment_operation, - fnumber, - ) - scaling_specification = parsed_scaling_specification + scaling_specification = parse_scaling_specification( + scaling_specification + ) # get the component sample and blocks from the asset model - for PG in PGB.index: + for pg in pgb.index: # noqa: PLR1702 # determine demand capacity adjustment operation, if required - cmp_loc_dir = '-'.join(PG[0:3]) - capacity_adjustment_operation = scaling_specification.get( - cmp_loc_dir, None + cmp_loc_dir = '-'.join(pg[0:3]) + capacity_adjustment_operation = scaling_specification.get( # type: ignore + cmp_loc_dir, ) - cmp_id = PG[0] - blocks = PGB.loc[PG, 'Blocks'] + cmp_id = pg[0] + blocks = pgb.loc[pg, 'Blocks'] # Calculate the block weights blocks = np.full(int(blocks), 1.0 / blocks) @@ -1284,54 +1452,82 @@ def map_ds(values, offset=int(ds_id + 1)): for val in frg_params.index.get_level_values(0).unique(): if 'LS' in val: - limit_states.append(val[2:]) + limit_states.append(val[2:]) # noqa: PERF401 ds_id = 0 - frg_rv_set_tags = [[] for b in blocks] - anchor_RVs = [] + frg_rv_set_tags: list = [[] for b in blocks] + anchor_rvs: list = [] for ls_id in limit_states: - frg_params_LS = frg_params[f'LS{ls_id}'] + frg_params_ls = frg_params[f'LS{ls_id}'] + + theta_0 = frg_params_ls.get('Theta_0', np.nan) + family = frg_params_ls.get('Family', 'deterministic') - theta_0 = frg_params_LS.get('Theta_0', np.nan) - family = frg_params_LS.get('Family', 'deterministic') - ds_weights = frg_params_LS.get('DamageStateWeights', np.nan) + # if `family` is defined but is `None`, we + # consider it to be `deterministic` + if not family: + family = 'deterministic' + ds_weights = frg_params_ls.get('DamageStateWeights', None) # check if the limit state is defined for the component if pd.isna(theta_0): continue - theta = [ - frg_params_LS.get(f"Theta_{t_i}", np.nan) for t_i in range(3) - ] + theta = np.array( + [ + value + for t_i in range(3) + if (value := frg_params_ls.get(f'Theta_{t_i}', None)) + is not None + ] + ) if capacity_adjustment_operation: - if family in {'normal', 'lognormal'}: - theta[0] = self._handle_operation( - theta[0], - capacity_adjustment_operation[0], - capacity_adjustment_operation[1], - ) + if family in {'normal', 'lognormal', 'deterministic'}: + # Only scale the median value if ls_id is defined in capacity_adjustment_operation + # Otherwise, use the original value + new_theta_0 = None + if 'ALL' in capacity_adjustment_operation: + new_theta_0 = self._handle_operation_list( + theta[0], + capacity_adjustment_operation['ALL'], + ) + elif f'LS{ls_id}' in capacity_adjustment_operation: + new_theta_0 = self._handle_operation_list( + theta[0], + capacity_adjustment_operation[f'LS{ls_id}'], + ) + if new_theta_0 is not None: + if new_theta_0.size == 1: + theta[0] = new_theta_0[0] + else: + # Repeat the theta values new_theta_0.size times along axis 0 + # and 1 time along axis 1 + theta = np.tile(theta, (new_theta_0.size, 1)) + theta[:, 0] = new_theta_0 else: - self.log.warn( + self.log.warning( f'Capacity adjustment is only supported ' f'for `normal` or `lognormal` distributions. ' f'Ignoring: `{cmp_loc_dir}`, which is `{family}`' ) - tr_lims = [ - frg_params_LS.get(f"Truncate{side}", np.nan) - for side in ("Lower", "Upper") - ] + tr_lims = np.array( + [ + frg_params_ls.get(f'Truncate{side}', np.nan) + for side in ('Lower', 'Upper') + ] + ) for block_i, _ in enumerate(blocks): frg_rv_tag = ( 'FRG-' - f'{PG[0]}-' # cmp_id - f'{PG[1]}-' # loc - f'{PG[2]}-' # dir - f'{PG[3]}-' # uid + f'{pg[0]}-' # cmp_id + f'{pg[1]}-' # loc + f'{pg[2]}-' # dir + f'{pg[3]}-' # uid f'{block_i + 1}-' # block f'{ls_id}' ) @@ -1352,11 +1548,11 @@ def map_ds(values, offset=int(ds_id + 1)): if ls_id == limit_states[0]: anchor = None else: - anchor = anchor_RVs[block_i] + anchor = anchor_rvs[block_i] # parse theta values for multilinear_CDF if family == 'multilinear_CDF': - theta = np.column_stack( + theta = np.column_stack( # type: ignore ( np.array( theta[0].split('|')[0].split(','), @@ -1369,55 +1565,55 @@ def map_ds(values, offset=int(ds_id + 1)): ) ) - RV = uq.rv_class_map(family)( + rv = uq.rv_class_map(family)( # type: ignore name=frg_rv_tag, theta=theta, truncation_limits=tr_lims, anchor=anchor, ) - capacity_RV_reg.add_RV(RV) + capacity_rv_reg.add_RV(rv) # type: ignore # add the RV to the set of correlated variables frg_rv_set_tags[block_i].append(frg_rv_tag) if ls_id == limit_states[0]: - anchor_RVs.append(RV) + anchor_rvs.append(rv) # Now add the LS->DS assignments lsds_rv_tag = ( 'LSDS-' - f'{PG[0]}-' # cmp_id - f'{PG[1]}-' # loc - f'{PG[2]}-' # dir - f'{PG[3]}-' # uid + f'{pg[0]}-' # cmp_id + f'{pg[1]}-' # loc + f'{pg[2]}-' # dir + f'{pg[3]}-' # uid f'{block_i + 1}-' # block f'{ls_id}' ) ds_id_next = assign_lsds( - ds_weights, ds_id, lsds_RV_reg, lsds_rv_tag + ds_weights, ds_id, lsds_rv_reg, lsds_rv_tag ) ds_id = ds_id_next if self._asmnt.log.verbose: - rv_count = len(lsds_RV_reg.RV) + rv_count = len(lsds_rv_reg.RV) self.log.msg( - f"2x{rv_count} random variables created.", prepend_timestamp=False + f'2x{rv_count} random variables created.', prepend_timestamp=False ) - return capacity_RV_reg, lsds_RV_reg + return capacity_rv_reg, lsds_rv_reg - def _prepare_dmg_quantities( + def prepare_dmg_quantities( self, component_sample: pd.DataFrame, - component_marginal_parameters: pd.DataFrame, + component_marginal_parameters: pd.DataFrame | None, + *, dropzero: bool = True, ) -> pd.DataFrame: """ - Combine component quantity and damage state information in one - DataFrame. + Combine component quantity and damage state information. This method assumes that a component quantity sample is available in the asset model and a damage state sample is @@ -1425,7 +1621,7 @@ def _prepare_dmg_quantities( Parameters ---------- - component_quantities: pd.DataFrame + component_sample: pd.DataFrame Component quantity sample from the AssetModel. component_marginal_parameters: pd.DataFrame Component marginal parameters from the AssetModel. @@ -1440,11 +1636,9 @@ def _prepare_dmg_quantities( damage state information. """ - # ('cmp', 'loc', 'dir', 'uid') -> component quantity series component_quantities = component_sample.to_dict('series') - # pylint: disable=missing-return-doc if self._asmnt.log.verbose: self.log.msg('Calculating damage quantities...', prepend_timestamp=True) @@ -1459,23 +1653,27 @@ def _prepare_dmg_quantities( # ('cmp', 'loc', 'dir', 'uid) -> number of blocks num_blocks = component_marginal_parameters['Blocks'].to_dict() - def get_num_blocks(key): - # pylint: disable=missing-return-type-doc + def get_num_blocks(key: object) -> float: return float(num_blocks[key]) else: # otherwise assume 1 block regardless of # ('cmp', 'loc', 'dir', 'uid) key - def get_num_blocks(_): - # pylint: disable=missing-return-type-doc + def get_num_blocks(key: object) -> float: # noqa: ARG001 return 1.00 # ('cmp', 'loc', 'dir', 'uid', 'block') -> damage state series + assert self.ds_sample is not None damage_state_sample_dict = self.ds_sample.to_dict('series') dmg_qnt_series_collection = {} for key, damage_state_series in damage_state_sample_dict.items(): - component, location, direction, uid, block = key + component: str + location: str + direction: str + uid: str + block: str + component, location, direction, uid, block = key # type: ignore damage_state_set = set(damage_state_series.values) for ds in damage_state_set: if ds == -1: @@ -1483,18 +1681,20 @@ def get_num_blocks(_): if dropzero and ds == 0: continue dmg_qnt_vals = np.where( - damage_state_series.values == ds, - component_quantities[component, location, direction, uid].values + damage_state_series.to_numpy() == ds, + component_quantities[ + component, location, direction, uid + ].to_numpy() / get_num_blocks((component, location, direction, uid)), 0.00, ) if -1 in damage_state_set: dmg_qnt_vals = np.where( - damage_state_series.values != -1, dmg_qnt_vals, np.nan + damage_state_series.to_numpy() != -1, dmg_qnt_vals, np.nan ) dmg_qnt_series = pd.Series(dmg_qnt_vals) dmg_qnt_series_collection[ - (component, location, direction, uid, block, str(ds)) + component, location, direction, uid, block, str(ds) ] = dmg_qnt_series damage_quantities = pd.concat( @@ -1507,13 +1707,11 @@ def get_num_blocks(_): # min_count=1 is specified so that the sum cross all NaNs will # result in NaN instead of zero. # https://stackoverflow.com/questions/33448003/sum-across-all-nans-in-pandas-returns-zero - damage_quantities = damage_quantities.groupby( + return damage_quantities.groupby( # type: ignore level=['cmp', 'loc', 'dir', 'uid', 'ds'], axis=1 ).sum(min_count=1) - return damage_quantities - - def _perform_dmg_task(self, task: list) -> None: + def perform_dmg_task(self, task: tuple) -> None: # noqa: C901 """ Perform a task from a damage process. @@ -1527,7 +1725,7 @@ def _perform_dmg_task(self, task: list) -> None: Parameters ---------- - task : list + task: list A list representing a task from the damage process. The list contains two elements: - The first element is a string representing the source @@ -1552,8 +1750,8 @@ def _perform_dmg_task(self, task: list) -> None: ValueError Raises an error if the source or target event descriptions do not follow expected formats. - """ + """ if self._asmnt.log.verbose: self.log.msg(f'Applying task {task}...', prepend_timestamp=True) @@ -1571,30 +1769,32 @@ def _perform_dmg_task(self, task: list) -> None: # check if the source component exists in the damage state # DataFrame + assert self.ds_sample is not None if source_cmp not in self.ds_sample.columns.get_level_values('cmp'): - self.log.warn( - f"Source component `{source_cmp}` in the prescribed " - "damage process not found among components in the damage " - "sample. The corresponding part of the damage process is " - "skipped." + self.log.warning( + f'Source component `{source_cmp}` in the prescribed ' + 'damage process not found among components in the damage ' + 'sample. The corresponding part of the damage process is ' + 'skipped.' ) return - # execute the events pres prescribed in the damage task + # execute the events prescribed in the damage task for source_event, target_infos in events.items(): # events can only be triggered by damage state occurrence if not source_event.startswith('DS'): - raise ValueError( - f"Unable to parse source event in damage " - f"process: `{source_event}`" + msg = ( + f'Unable to parse source event in damage ' + f'process: `{source_event}`' ) + raise ValueError(msg) # get the ID of the damage state that triggers the event ds_source = int(source_event[2:]) # turn the target_infos into a list if it is a single # argument, for consistency if not isinstance(target_infos, list): - target_infos = [target_infos] + target_infos = [target_infos] # noqa: PLW2901 for target_info in target_infos: # get the target component and event type @@ -1603,11 +1803,11 @@ def _perform_dmg_task(self, task: list) -> None: if (target_cmp != 'ALL') and ( target_cmp not in self.ds_sample.columns.get_level_values('cmp') ): - self.log.warn( - f"Target component `{target_cmp}` in the prescribed " - "damage process not found among components in the damage " - "sample. The corresponding part of the damage process is " - "skipped." + self.log.warning( + f'Target component `{target_cmp}` in the prescribed ' + 'damage process not found among components in the damage ' + 'sample. The corresponding part of the damage process is ' + 'skipped.' ) continue @@ -1623,10 +1823,11 @@ def _perform_dmg_task(self, task: list) -> None: # -1 stands for nan (ints don'ts support nan) else: - raise ValueError( - f"Unable to parse target event in damage " - f"process: `{target_event}`" + msg = ( + f'Unable to parse target event in damage ' + f'process: `{target_event}`' ) + raise ValueError(msg) if match_locations: self._perform_dmg_event_loc( @@ -1648,15 +1849,15 @@ def _perform_dmg_event( ) -> None: """ Perform a damage event. - See `_perform_dmg_task`. + See `_perform_dmg_task`. """ - # affected rows + assert self.ds_sample is not None row_selection = np.where( # for many instances of source_cmp, we # consider the highest damage state - self.ds_sample[source_cmp].max(axis=1).values + self.ds_sample[source_cmp].max(axis=1).to_numpy() # type: ignore == ds_source )[0] # affected columns @@ -1675,19 +1876,32 @@ def _perform_dmg_event_loc( ) -> None: """ Perform a damage event matching locations. - See `_perform_dmg_task`. - """ + Parameters + ---------- + source_cmp: str + Source component, e.g., `'1_CMP_A'`. The number in the beginning + is used to order the tasks and is not considered here. + ds_source: int + Source damage state. + target_cmp: str + Target component, e.g., `'CMP_B'`. The components that + will be affected when `source_cmp` gets to `ds_source`. + ds_target: int + Target damage state, e.g., `'DS_1'`. The damage state that + is assigned to `target_cmp` when `source_cmp` gets to + `ds_source`. + """ # get locations of source component + assert self.ds_sample is not None source_locs = set(self.ds_sample[source_cmp].columns.get_level_values('loc')) for loc in source_locs: # apply damage task matching locations row_selection = np.where( # for many instances of source_cmp, we # consider the highest damage state - self.ds_sample[source_cmp, loc].max(axis=1).values - == ds_source + self.ds_sample[source_cmp, loc].max(axis=1).to_numpy() == ds_source )[0] # affected columns @@ -1707,14 +1921,16 @@ def _perform_dmg_event_loc( )[0] self.ds_sample.iloc[row_selection, column_selection] = ds_target - def _complete_ds_cols(self, dmg_sample: pd.DataFrame) -> pd.DataFrame: + def complete_ds_cols(self, dmg_sample: pd.DataFrame) -> pd.DataFrame: """ + Complete damage state columns. + Completes the damage sample DataFrame with all possible damage states for each component. Parameters ---------- - dmg_sample : DataFrame + dmg_sample: DataFrame A DataFrame containing the damage state information for each component block in the asset model. The columns are MultiIndexed with levels corresponding to component @@ -1739,32 +1955,37 @@ def _complete_ds_cols(self, dmg_sample: pd.DataFrame) -> pd.DataFrame: damage states for each component. """ - # get a shortcut for the damage model parameters - DP = self.damage_params + dp = self.damage_params + assert dp is not None # Get the header for the results that we can use to identify # cmp-loc-dir-uid sets dmg_header = ( - dmg_sample.groupby(level=[0, 1, 2, 3], axis=1).first().iloc[:2, :] + dmg_sample.groupby( # type: ignore + level=[0, 1, 2, 3], + axis=1, + ) + .first() + .iloc[:2, :] ) damaged_components = set(dmg_header.columns.get_level_values('cmp')) # get the number of possible limit states - ls_list = [col for col in DP.columns.unique(level=0) if 'LS' in col] + ls_list = [col for col in dp.columns.unique(level=0) if 'LS' in col] # initialize the result DataFrame res = pd.DataFrame() - # TODO: For the code below, store the number of damage states + # TODO(JVM): For the code below, store the number of damage states # for each component ID as an attribute of the ds_model when # loading the parameters, and then directly access them here # much faster instead of parsing the parameters again. # walk through all components that have damage parameters provided - for cmp_id in DP.index: + for cmp_id in dp.index: # get the component-specific parameters - cmp_data = DP.loc[cmp_id] + cmp_data = dp.loc[cmp_id] # and initialize the damage state counter ds_count = 0 @@ -1772,15 +1993,15 @@ def _complete_ds_cols(self, dmg_sample: pd.DataFrame) -> pd.DataFrame: # walk through all limit states for the component for ls in ls_list: # check if the given limit state is defined - if not pd.isna(cmp_data[(ls, 'Theta_0')]): + if not pd.isna(cmp_data[ls, 'Theta_0']): # check if there is only one damage state - if pd.isna(cmp_data[(ls, 'DamageStateWeights')]): + if pd.isna(cmp_data[ls, 'DamageStateWeights']): ds_count += 1 else: # or if there are more than one, how many ds_count += len( - cmp_data[(ls, 'DamageStateWeights')].split('|') + cmp_data[ls, 'DamageStateWeights'].split('|') ) # get the list of valid cmp-loc-dir-uid sets @@ -1793,7 +2014,7 @@ def _complete_ds_cols(self, dmg_sample: pd.DataFrame) -> pd.DataFrame: # multiindexed column cmp_headers = pd.concat( [cmp_header for ds_i in range(ds_count + 1)], - keys=[str(r) for r in range(0, ds_count + 1)], + keys=[str(r) for r in range(ds_count + 1)], axis=1, ) cmp_headers.columns.names = ['ds', *cmp_headers.columns.names[1::]] @@ -1805,7 +2026,7 @@ def _complete_ds_cols(self, dmg_sample: pd.DataFrame) -> pd.DataFrame: # the damage states at the lowest like - matching the dmg_sample input res = pd.DataFrame( 0.0, - columns=res.columns.reorder_levels([1, 2, 3, 4, 0]), + columns=res.columns.reorder_levels([1, 2, 3, 4, 0]), # type: ignore index=dmg_sample.index, ) @@ -1817,7 +2038,20 @@ def _complete_ds_cols(self, dmg_sample: pd.DataFrame) -> pd.DataFrame: def _is_for_ds_model(data: pd.DataFrame) -> bool: """ + Check if data are for `ds_model`. + Determines if the specified damage model parameters are for components modeled with discrete Damage States (DS). + + Parameters + ---------- + data: pd.DataFrame + The data to check. + + Returns + ------- + bool + If the data are for `ds_model`. + """ return 'LS1' in data.columns.get_level_values(0) diff --git a/pelicun/model/demand_model.py b/pelicun/model/demand_model.py index b5ceefab0..c6bee0e36 100644 --- a/pelicun/model/demand_model.py +++ b/pelicun/model/demand_model.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -38,32 +37,25 @@ # Adam Zsarnóczay # John Vouvakis Manousakis -""" -This file defines the DemandModel object and its methods. -.. rubric:: Contents - -.. autosummary:: - - DemandModel - -""" +"""DemandModel object and associated methods.""" from __future__ import annotations -from typing import TYPE_CHECKING + import re -import os from collections import defaultdict +from pathlib import Path +from typing import TYPE_CHECKING, overload + +import numexpr as ne import numpy as np import pandas as pd -import numexpr as ne + +from pelicun import base, file_io, uq from pelicun.model.pelicun_model import PelicunModel -from pelicun import base -from pelicun import uq -from pelicun import file_io if TYPE_CHECKING: - from pelicun.assessment import Assessment + from pelicun.assessment import AssessmentBase idx = base.idx @@ -102,32 +94,49 @@ class DemandModel(PelicunModel): """ __slots__ = [ - 'marginal_params', + '_RVs', + 'calibrated', 'correlation', 'empirical_data', - 'user_units', - 'calibrated', - '_RVs', + 'marginal_params', 'sample', + 'user_units', ] - def __init__(self, assessment: Assessment): + def __init__(self, assessment: AssessmentBase) -> None: + """ + Instantiate a DemandModel. + + Parameters + ---------- + assessment: Assessment + Parent assessment object. + + """ super().__init__(assessment) - self.marginal_params = None - self.correlation = None - self.empirical_data = None - self.user_units = None + self.marginal_params: pd.DataFrame | None = None + self.correlation: pd.DataFrame | None = None + self.empirical_data: pd.DataFrame | None = None + self.user_units: pd.Series | None = None self.calibrated = False - self._RVs = None - self.sample = None + self._RVs: uq.RandomVariableRegistry | None = None + self.sample: pd.DataFrame | None = None + + @overload + def save_sample( + self, filepath: None = None, *, save_units: bool = False + ) -> tuple[pd.DataFrame, pd.Series] | pd.DataFrame: ... + + @overload + def save_sample(self, filepath: str, *, save_units: bool = False) -> None: ... def save_sample( - self, filepath: str | None = None, save_units: bool = False - ) -> None | tuple[pd.DataFrame, pd.Series]: + self, filepath: str | None = None, *, save_units: bool = False + ) -> None | tuple[pd.DataFrame, pd.Series] | pd.DataFrame: """ - Save demand sample to a csv file or return it in a DataFrame + Save demand sample to a csv file or return it in a DataFrame. Returns ------- @@ -139,26 +148,20 @@ def save_sample( If `save_units` is True, it returns a tuple of the DataFrame and a Series containing the units. - Raises - ------ - IOError - Raises an IOError if there is an issue saving the file to - the specified `filepath`. """ - self.log.div() if filepath is not None: self.log.msg('Saving demand sample...') + assert self.sample is not None res = file_io.save_to_csv( self.sample, - filepath, + Path(filepath) if filepath is not None else None, units=self.user_units, unit_conversion_factors=self._asmnt.unit_conversion_factors, use_simpleindex=(filepath is not None), log=self._asmnt.log, ) - if filepath is not None: self.log.msg( 'Demand sample successfully saved.', prepend_timestamp=False @@ -166,8 +169,11 @@ def save_sample( return None # else: - units = res.loc["Units"] - res.drop("Units", inplace=True) + assert isinstance(res, pd.DataFrame) + + units = res.loc['Units'] + res = res.drop('Units') + assert isinstance(units, pd.Series) if save_units: return res.astype(float), units @@ -190,8 +196,10 @@ def load_sample(self, filepath: str | pd.DataFrame) -> None: """ - def parse_header(raw_header): + def parse_header(raw_header: pd.Index[str]) -> pd.Index[str]: """ + Parse and clean header. + Parses and cleans the header of a demand DataFrame from raw multi-level index to a standardized format. @@ -206,7 +214,7 @@ def parse_header(raw_header): Parameters ---------- - raw_header : pd.MultiIndex + raw_header: pd.MultiIndex The original multi-level index (header) of the DataFrame, which may contain an optional event_ID and might have excess whitespace in the labels. @@ -219,24 +227,26 @@ def parse_header(raw_header): index has three levels: 'type', 'loc', and 'dir', representing the type of demand, location, and direction, respectively. + """ - old_MI = raw_header + old_mi = raw_header # The first number (event_ID) in the demand labels is optional and # currently not used. We remove it if it was in the raw data. - if old_MI.nlevels == 4: + num_levels_with_event_id = 4 + if old_mi.nlevels == num_levels_with_event_id: if self._asmnt.log.verbose: self.log.msg( 'Removing event_ID from header...', prepend_timestamp=False ) new_column_index_array = np.array( - [old_MI.get_level_values(i) for i in range(1, 4)] + [old_mi.get_level_values(i) for i in range(1, 4)] ) else: new_column_index_array = np.array( - [old_MI.get_level_values(i) for i in range(3)] + [old_mi.get_level_values(i) for i in range(3)] ) # Remove whitespace to avoid ambiguity @@ -252,12 +262,10 @@ def parse_header(raw_header): # Creating new, cleaned-up header - new_MI = pd.MultiIndex.from_arrays( + return pd.MultiIndex.from_arrays( new_column_index, names=['type', 'loc', 'dir'] ) - return new_MI - self.log.div() self.log.msg('Loading demand data...') @@ -267,6 +275,8 @@ def parse_header(raw_header): return_units=True, log=self._asmnt.log, ) + assert isinstance(demand_data, pd.DataFrame) + assert isinstance(units, pd.Series) parsed_data = demand_data.copy() @@ -280,14 +290,23 @@ def parse_header(raw_header): 'Removing errors from the raw data...', prepend_timestamp=False ) - error_list = parsed_data.loc[:, idx['ERROR', :, :]].values.astype(bool) + error_list = ( + parsed_data.loc[ # type: ignore + :, # type: ignore + idx['ERROR', :, :], # type: ignore + ] + .to_numpy() + .astype( # type: ignore + bool # type: ignore + ) + ) # type: ignore parsed_data = parsed_data.loc[~error_list, :].copy() - parsed_data.drop('ERROR', level=0, axis=1, inplace=True) + parsed_data = parsed_data.drop('ERROR', level=0, axis=1) self.log.msg( - "\nBased on the values in the ERROR column, " - f"{np.sum(error_list)} demand samples were removed.\n", + '\nBased on the values in the ERROR column, ' + f'{np.sum(error_list)} demand samples were removed.\n', prepend_timestamp=False, ) @@ -302,10 +321,15 @@ def parse_header(raw_header): self.log.msg('Demand units successfully parsed.', prepend_timestamp=False) - def estimate_RID( - self, demands: pd.DataFrame, params: dict, method: str = 'FEMA P58' + def estimate_RID( # noqa: N802 + self, + demands: pd.DataFrame | pd.Series, + params: dict, + method: str = 'FEMA P58', ) -> pd.DataFrame: """ + Estimate residual inter-story drift (RID). + Estimates residual inter-story drift (RID) realizations based on peak inter-story drift (PID) and other demand parameters using specified methods. @@ -318,16 +342,16 @@ def estimate_RID( Parameters ---------- - demands : DataFrame + demands: DataFrame A DataFrame containing samples of demands, specifically peak inter-story drift (PID) values for various location-direction pairs required for the estimation method. - params : dict + params: dict A dictionary containing parameters required for the estimation method, such as 'yield_drift', which is the drift at which yielding is expected to occur. - method : str, optional + method: str, optional The method used to estimate the RID values. Currently, only 'FEMA P58' is implemented. Defaults to 'FEMA P58'. @@ -353,6 +377,7 @@ def estimate_RID( RID values to model the inherent uncertainty. The method ensures that the RID values do not exceed the corresponding PID values. + """ if method in {'FEMA P58', 'FEMA P-58'}: # method is described in FEMA P-58 Volume 1 Section 5.4 & @@ -360,61 +385,63 @@ def estimate_RID( # the provided demands shall be PID values at various # loc-dir pairs - PID = demands + pid = demands # there's only one parameter needed: the yield drift yield_drift = params['yield_drift'] # three subdomains of demands are identified - small = PID < yield_drift - medium = PID < 4 * yield_drift - large = PID >= 4 * yield_drift + small = yield_drift > pid + medium = 4 * yield_drift > pid + large = 4 * yield_drift <= pid # convert PID to RID in each subdomain - RID = PID.copy() - RID[large] = PID[large] - 3 * yield_drift - RID[medium] = 0.3 * (PID[medium] - yield_drift) - RID[small] = 0.0 + rid = pid.copy() + rid[large] = pid[large] - 3 * yield_drift + rid[medium] = 0.3 * (pid[medium] - yield_drift) + rid[small] = 0.0 # add extra uncertainty to nonzero values rng = self._asmnt.options.rng - eps = rng.normal(scale=0.2, size=RID.shape) - RID[RID > 0] = np.exp(np.log(RID[RID > 0]) + eps) + eps = rng.normal(scale=0.2, size=rid.shape) + rid[rid > 0] = np.exp(np.log(rid[rid > 0]) + eps) # type: ignore # finally, make sure the RID values are never larger than # the PIDs - RID = pd.DataFrame( - np.minimum(PID.values, RID.values), - columns=pd.DataFrame( + rid = pd.DataFrame( + np.minimum(pid.values, rid.values), # type: ignore + columns=pd.DataFrame( # noqa: PD013 1, index=['RID'], - columns=PID.columns, + columns=pid.columns, ) .stack(level=[0, 1]) .index, - index=PID.index, + index=pid.index, ) else: - RID = None + msg = f'Invalid method: `{method}`.' + raise ValueError(msg) - # return the generated drift realizations - return RID + return rid - def estimate_RID_and_adjust_sample( + def estimate_RID_and_adjust_sample( # noqa: N802 self, params: dict, method: str = 'FEMA P58' ) -> None: """ + Estimate residual inter-story drift (RID) and modifies sample. + Uses `self.estimate_RID` and adjusts the demand sample. See the docstring of the `estimate_RID` method for details. Parameters ---------- - params : dict + params: dict A dictionary containing parameters required for the estimation method, such as 'yield_drift', which is the drift at which yielding is expected to occur. - method : str, optional + method: str, optional The method used to estimate the RID values. Currently, only 'FEMA P58' is implemented. Defaults to 'FEMA P58'. @@ -424,14 +451,18 @@ def estimate_RID_and_adjust_sample( If the method is called before a sample is generated. """ - if self.sample is None: - raise ValueError('Demand model does not have a sample yet.') - - demand_sample, demand_units = self.save_sample(save_units=True) + msg = 'Demand model does not have a sample yet.' + raise ValueError(msg) + + sample_tuple = self.save_sample(save_units=True) + assert isinstance(sample_tuple, tuple) + demand_sample, demand_units = sample_tuple + assert isinstance(demand_sample, pd.DataFrame) + assert isinstance(demand_units, pd.Series) pid = demand_sample['PID'] rid = self.estimate_RID(pid, params, method) - rid_units = pd.Series('rad', index=rid.columns) + rid_units = pd.Series('unitless', index=rid.columns) demand_sample_ext = pd.concat([demand_sample, rid], axis=1) units_ext = pd.concat([demand_units, rid_units]) demand_sample_ext.loc['Units', :] = units_ext @@ -440,15 +471,15 @@ def estimate_RID_and_adjust_sample( def expand_sample( self, label: str, - value: float, + value: float | np.ndarray, unit: str, - location='0', - direction='1' + location: str = '0', + direction: str = '1', ) -> None: """ - Adds an extra column to the demand sample. + Add an extra column to the demand sample. - The column comtains repeated instances of `value`, is accessed + The column contains repeated instances of `value`, is accessed via the multi-index (`label`-`location`-`direction`), and has units of `unit`. @@ -456,7 +487,7 @@ def expand_sample( ---------- label: str Label to use to extend the MultiIndex of the demand sample. - value: float + value: float | np.ndarray Values to add to the rows of the additional column. unit: str Unit that corresponds to the additional column. @@ -469,20 +500,29 @@ def expand_sample( ------ ValueError If the method is called before a sample is generated. + ValueError + If `value` is a numpy array of incorrect shape. """ if self.sample is None: - raise ValueError('Demand model does not have a sample yet.') - demand_sample, demand_units = self.save_sample(save_units=True) - demand_sample[(label, location, direction)] = value - demand_units[(label, location, direction)] = unit + msg = 'Demand model does not have a sample yet.' + raise ValueError(msg) + sample_tuple = self.save_sample(save_units=True) + assert isinstance(sample_tuple, tuple) + demand_sample, demand_units = sample_tuple + assert isinstance(demand_sample, pd.DataFrame) + assert isinstance(demand_units, pd.Series) + if isinstance(value, np.ndarray) and len(value) != len(demand_sample): + msg = 'Incompatible array length.' + raise ValueError(msg) + demand_sample[label, location, direction] = value + demand_units[label, location, direction] = unit demand_sample.loc['Units', :] = demand_units self.load_sample(demand_sample) - - def calibrate_model(self, config: dict) -> None: + def calibrate_model(self, config: dict) -> None: # noqa: C901 """ - Calibrate a demand model to describe the raw demand data + Calibrate a demand model to describe the raw demand data. The raw data shall be parsed first to ensure that it follows the schema expected by this method. The calibration settings define the @@ -497,27 +537,30 @@ def calibrate_model(self, config: dict) -> None: settings for the calibration. """ - if self.calibrated: - self.log.warn('DemandModel has been previously calibrated.') + self.log.warning('DemandModel has been previously calibrated.') - def parse_settings(settings, demand_type): - def parse_str_to_float(in_str, context_string): - # pylint: disable = missing-return-type-doc - # pylint: disable = missing-return-doc + def parse_settings( # noqa: C901 + cal_df: pd.DataFrame, settings: dict, demand_type: str + ) -> None: + def parse_str_to_float(in_str: str, context_string: str) -> float: try: - out_float = float(in_str) + out_float = ( + np.nan if base.check_if_str_is_na(in_str) else float(in_str) + ) except ValueError: - self.log.warn( - f"Could not parse {in_str} provided as " - f"{context_string}. Using NaN instead." + self.log.warning( + f'Could not parse {in_str} provided as ' + f'{context_string}. Using NaN instead.' ) out_float = np.nan return out_float + demand_sample = self.save_sample() + assert isinstance(demand_sample, pd.DataFrame) active_d_types = demand_sample.columns.get_level_values('type').unique() if demand_type == 'ALL': @@ -528,12 +571,12 @@ def parse_str_to_float(in_str, context_string): for d_type in active_d_types: if d_type.split('_')[0] == demand_type: - cols_lst.append(d_type) + cols_lst.append(d_type) # noqa: PERF401 cols = tuple(cols_lst) # load the distribution family - cal_df.loc[idx[cols, :, :], 'Family'] = settings['DistributionFamily'] + cal_df.loc[list(cols), 'Family'] = settings['DistributionFamily'] # load limits for lim in ( @@ -542,13 +585,13 @@ def parse_str_to_float(in_str, context_string): 'TruncateLower', 'TruncateUpper', ): - if lim in settings.keys(): + if lim in settings: val = parse_str_to_float(settings[lim], lim) if not pd.isna(val): - cal_df.loc[idx[cols, :, :], lim] = val + cal_df.loc[list(cols), lim] = val # scale the censor and truncation limits, if needed - scale_factor = self._asmnt.scale_factor(settings.get('Unit', None)) + scale_factor = self._asmnt.scale_factor(settings.get('Unit')) rows_to_scale = [ 'CensorLower', @@ -556,10 +599,10 @@ def parse_str_to_float(in_str, context_string): 'TruncateLower', 'TruncateUpper', ] - cal_df.loc[idx[cols, :, :], rows_to_scale] *= scale_factor + cal_df.loc[idx[cols, :, :], rows_to_scale] *= scale_factor # type: ignore # load the prescribed additional uncertainty - if 'AddUncertainty' in settings.keys(): + if 'AddUncertainty' in settings: sig_increase = parse_str_to_float( settings['AddUncertainty'], 'AddUncertainty' ) @@ -568,11 +611,13 @@ def parse_str_to_float(in_str, context_string): if settings['DistributionFamily'] == 'normal': sig_increase *= scale_factor - cal_df.loc[idx[cols, :, :], 'SigIncrease'] = sig_increase + cal_df.loc[list(cols), 'SigIncrease'] = sig_increase - def get_filter_mask(lower_lims, upper_lims): - # pylint: disable=missing-return-doc - # pylint: disable=missing-return-type-doc + def get_filter_mask( + demand_sample: pd.DataFrame, + lower_lims: np.ndarray, + upper_lims: np.ndarray, + ) -> bool: demands_of_interest = demand_sample.iloc[:, pd.notna(upper_lims)] limits_of_interest = upper_lims[pd.notna(upper_lims)] upper_mask = np.all(demands_of_interest < limits_of_interest, axis=1) @@ -587,6 +632,7 @@ def get_filter_mask(lower_lims, upper_lims): self.log.msg('Calibrating demand model...') demand_sample = self.sample + assert isinstance(demand_sample, pd.DataFrame) # initialize a DataFrame that contains calibration information cal_df = pd.DataFrame( @@ -607,21 +653,21 @@ def get_filter_mask(lower_lims, upper_lims): cal_df['Family'] = cal_df['Family'].astype(str) # start by assigning the default option ('ALL') to every demand column - parse_settings(config['ALL'], 'ALL') + parse_settings(cal_df, config['ALL'], demand_type='ALL') # then parse the additional settings and make the necessary adjustments - for demand_type in config.keys(): + for demand_type in config: # noqa: PLC0206 if demand_type != 'ALL': - parse_settings(config[demand_type], demand_type) + parse_settings(cal_df, config[demand_type], demand_type=demand_type) if self._asmnt.log.verbose: self.log.msg( - "\nCalibration settings successfully parsed:\n" + str(cal_df), + '\nCalibration settings successfully parsed:\n' + str(cal_df), prepend_timestamp=False, ) else: self.log.msg( - "\nCalibration settings successfully parsed:\n", + '\nCalibration settings successfully parsed:\n', prepend_timestamp=False, ) @@ -632,18 +678,19 @@ def get_filter_mask(lower_lims, upper_lims): # Currently, non-empirical demands are assumed to have some level of # correlation, hence, a censored value in any demand triggers the # removal of the entire sample from the population. - upper_lims = cal_df.loc[:, 'CensorUpper'].values - lower_lims = cal_df.loc[:, 'CensorLower'].values + upper_lims = cal_df.loc[:, 'CensorUpper'].to_numpy() + lower_lims = cal_df.loc[:, 'CensorLower'].to_numpy() + assert isinstance(demand_sample, pd.DataFrame) if ~np.all(pd.isna(np.array([upper_lims, lower_lims]))): - censor_mask = get_filter_mask(lower_lims, upper_lims) + censor_mask = get_filter_mask(demand_sample, lower_lims, upper_lims) censored_count = np.sum(~censor_mask) - demand_sample = demand_sample.loc[censor_mask, :] + demand_sample = pd.DataFrame(demand_sample.loc[censor_mask, :]) self.log.msg( - "\nBased on the provided censoring limits, " - f"{censored_count} samples were censored.", + '\nBased on the provided censoring limits, ' + f'{censored_count} samples were censored.', prepend_timestamp=False, ) else: @@ -653,20 +700,21 @@ def get_filter_mask(lower_lims, upper_lims): # If yes, that suggests an error either in the samples or the # configuration. We handle such errors gracefully: the analysis is not # terminated, but we show an error in the log file. - upper_lims = cal_df.loc[:, 'TruncateUpper'].values - lower_lims = cal_df.loc[:, 'TruncateLower'].values + upper_lims = cal_df.loc[:, 'TruncateUpper'].to_numpy() + lower_lims = cal_df.loc[:, 'TruncateLower'].to_numpy() + assert isinstance(demand_sample, pd.DataFrame) if ~np.all(pd.isna(np.array([upper_lims, lower_lims]))): - truncate_mask = get_filter_mask(lower_lims, upper_lims) + truncate_mask = get_filter_mask(demand_sample, lower_lims, upper_lims) truncated_count = np.sum(~truncate_mask) if truncated_count > 0: - demand_sample = demand_sample.loc[truncate_mask, :] + demand_sample = pd.DataFrame(demand_sample.loc[truncate_mask, :]) self.log.msg( - "\nBased on the provided truncation limits, " - f"{truncated_count} samples were removed before demand " - "calibration.", + '\nBased on the provided truncation limits, ' + f'{truncated_count} samples were removed before demand ' + 'calibration.', prepend_timestamp=False, ) @@ -677,8 +725,9 @@ def get_filter_mask(lower_lims, upper_lims): empirical_edps = [] for edp in cal_df.index: if cal_df.loc[edp, 'Family'] == 'empirical': - empirical_edps.append(edp) + empirical_edps.append(edp) # noqa: PERF401 + assert isinstance(demand_sample, pd.DataFrame) if empirical_edps: self.empirical_data = demand_sample.loc[:, empirical_edps].copy() @@ -690,22 +739,25 @@ def get_filter_mask(lower_lims, upper_lims): if self._asmnt.log.verbose: self.log.msg( - f"\nDemand data used for calibration:\n{demand_sample}", + f'\nDemand data used for calibration:\n{demand_sample}', prepend_timestamp=False, ) # fit the joint distribution self.log.msg( - "\nFitting the prescribed joint demand distribution...", + '\nFitting the prescribed joint demand distribution...', prepend_timestamp=False, ) demand_theta, demand_rho = uq.fit_distribution_to_sample( - raw_samples=demand_sample.values.T, - distribution=cal_df.loc[:, 'Family'].values, + raw_sample=demand_sample.to_numpy().T, + distribution=cal_df.loc[:, 'Family'].values, # type: ignore censored_count=censored_count, - detection_limits=cal_df.loc[:, ['CensorLower', 'CensorUpper']].values, - truncation_limits=cal_df.loc[ + detection_limits=cal_df.loc[ # type: ignore + :, + ['CensorLower', 'CensorUpper'], + ].values, + truncation_limits=cal_df.loc[ # type: ignore :, ['TruncateLower', 'TruncateUpper'] ].values, multi_fit=False, @@ -713,7 +765,7 @@ def get_filter_mask(lower_lims, upper_lims): ) # fit the joint distribution self.log.msg( - "\nCalibration successful, processing results...", + '\nCalibration successful, processing results...', prepend_timestamp=False, ) @@ -722,12 +774,16 @@ def get_filter_mask(lower_lims, upper_lims): # increase the variance of the marginal distributions, if needed if ~np.all(pd.isna(model_params.loc[:, 'SigIncrease'].values)): - self.log.msg("\nIncreasing demand variance...", prepend_timestamp=False) + self.log.msg('\nIncreasing demand variance...', prepend_timestamp=False) - sig_inc = np.nan_to_num(model_params.loc[:, 'SigIncrease'].values) - sig_0 = model_params.loc[:, 'Theta_1'].values + sig_inc = np.nan_to_num( + model_params.loc[:, 'SigIncrease'].values, # type: ignore + ) + sig_0 = model_params.loc[:, 'Theta_1'].to_numpy() - model_params.loc[:, 'Theta_1'] = np.sqrt(sig_0**2.0 + sig_inc**2.0) + model_params.loc[:, 'Theta_1'] = np.sqrt( + sig_0**2.0 + sig_inc**2.0, # type: ignore + ) # remove unneeded fields from model_params for col in ('SigIncrease', 'CensorLower', 'CensorUpper'): @@ -741,7 +797,7 @@ def get_filter_mask(lower_lims, upper_lims): self.marginal_params = model_params self.log.msg( - "\nCalibrated demand model marginal distributions:\n" + '\nCalibrated demand model marginal distributions:\n' + str(model_params), prepend_timestamp=False, ) @@ -752,7 +808,7 @@ def get_filter_mask(lower_lims, upper_lims): ) self.log.msg( - "\nCalibrated demand model correlation matrix:\n" + '\nCalibrated demand model correlation matrix:\n' + str(self.correlation), prepend_timestamp=False, ) @@ -760,20 +816,16 @@ def get_filter_mask(lower_lims, upper_lims): self.calibrated = True def save_model(self, file_prefix: str) -> None: - """ - Save parameters of the demand model to a set of csv files - - """ - + """Save parameters of the demand model to a set of csv files.""" self.log.div() self.log.msg('Saving demand model...') # save the correlation and empirical data - file_io.save_to_csv(self.correlation, file_prefix + '_correlation.csv') + file_io.save_to_csv(self.correlation, Path(file_prefix + '_correlation.csv')) if self.empirical_data is not None: file_io.save_to_csv( self.empirical_data, - file_prefix + '_empirical.csv', + Path(file_prefix + '_empirical.csv'), units=self.user_units, unit_conversion_factors=self._asmnt.unit_conversion_factors, log=self._asmnt.log, @@ -782,6 +834,8 @@ def save_model(self, file_prefix: str) -> None: # Converting the marginal parameters requires special # treatment, so we can't rely on file_io's universal unit # conversion functionality. We do it manually here instead. + assert isinstance(self.marginal_params, pd.DataFrame) + assert isinstance(self.user_units, pd.Series) marginal_params_user_units = self._convert_marginal_params( self.marginal_params.copy(), self.user_units, inverse_conversion=True ) @@ -789,7 +843,7 @@ def save_model(self, file_prefix: str) -> None: file_io.save_to_csv( marginal_params_user_units, - file_prefix + '_marginals.csv', + Path(file_prefix + '_marginals.csv'), orientation=1, log=self._asmnt.log, ) @@ -809,43 +863,61 @@ def load_model(self, data_source: str | dict) -> None: _correlation.csv. If dict, the data source is a dictionary with the following optional keys: 'marginals', 'empirical', and 'correlation'. The value under each key shall be a DataFrame. - """ + Raises + ------ + TypeError + If the data source type is invalid. + + """ self.log.div() self.log.msg('Loading demand model...') # prepare the marginal data source variable to load the data if isinstance(data_source, dict): marginal_data_source = data_source.get('marginals') + assert isinstance(marginal_data_source, pd.DataFrame) empirical_data_source = data_source.get('empirical', None) correlation_data_source = data_source.get('correlation', None) - else: + elif isinstance(data_source, str): marginal_data_source = data_source + '_marginals.csv' empirical_data_source = data_source + '_empirical.csv' correlation_data_source = data_source + '_correlation.csv' + else: + msg = f'Invalid data_source type: {type(data_source)}.' + raise TypeError(msg) if empirical_data_source is not None: - if isinstance(empirical_data_source, str) and os.path.exists( - empirical_data_source + if ( + isinstance(empirical_data_source, str) + and Path(empirical_data_source).exists() ): - self.empirical_data = file_io.load_data( + empirical_data = file_io.load_data( empirical_data_source, self._asmnt.unit_conversion_factors, log=self._asmnt.log, ) - self.empirical_data.columns.names = ('type', 'loc', 'dir') + assert isinstance(empirical_data, pd.DataFrame) + self.empirical_data = empirical_data + self.empirical_data.columns.names = ['type', 'loc', 'dir'] else: self.empirical_data = None if correlation_data_source is not None: - self.correlation = file_io.load_data( + correlation = file_io.load_data( correlation_data_source, self._asmnt.unit_conversion_factors, reindex=False, log=self._asmnt.log, ) - self.correlation.index.set_names(['type', 'loc', 'dir'], inplace=True) - self.correlation.columns.set_names(['type', 'loc', 'dir'], inplace=True) + assert isinstance(correlation, pd.DataFrame) + self.correlation = correlation + self.correlation.index = self.correlation.index.set_names( + ['type', 'loc', 'dir'] + ) + self.correlation.columns = self.correlation.columns.set_names( + ['type', 'loc', 'dir'] + ) else: self.correlation = None @@ -858,7 +930,12 @@ def load_model(self, data_source: str | dict) -> None: return_units=True, log=self._asmnt.log, ) - marginal_params.index.set_names(['type', 'loc', 'dir'], inplace=True) + assert isinstance(marginal_params, pd.DataFrame) + assert isinstance(units, pd.Series) + + marginal_params.index = marginal_params.index.set_names( + ['type', 'loc', 'dir'] + ) marginal_params = self._convert_marginal_params( marginal_params.copy(), units @@ -869,53 +946,59 @@ def load_model(self, data_source: str | dict) -> None: self.log.msg('Demand model successfully loaded.', prepend_timestamp=False) - def _create_RVs(self, preserve_order: bool = False) -> None: - """ - Create a random variable registry for the joint distribution of demands. - - """ + def _create_RVs(self, *, preserve_order: bool = False) -> None: # noqa: N802 + """Create a random variable registry for the joint distribution of demands.""" + assert self.marginal_params is not None # initialize the registry - RV_reg = uq.RandomVariableRegistry(self._asmnt.options.rng) + rv_reg = uq.RandomVariableRegistry(self._asmnt.options.rng) # add a random variable for each demand variable for rv_params in self.marginal_params.itertuples(): edp = rv_params.Index - rv_tag = f'EDP-{edp[0]}-{edp[1]}-{edp[2]}' - family = getattr(rv_params, "Family", 'deterministic') + rv_tag = f'EDP-{edp[0]}-{edp[1]}-{edp[2]}' # type: ignore + family = getattr(rv_params, 'Family', 'deterministic') if family == 'empirical': - if preserve_order: - dist_family = 'coupled_empirical' - else: - dist_family = 'empirical' + dist_family = 'coupled_empirical' if preserve_order else 'empirical' # empirical RVs need the data points - RV_reg.add_RV( + rv_reg.add_RV( uq.rv_class_map(dist_family)( name=rv_tag, - raw_samples=self.empirical_data.loc[:, edp].values, + theta=self.empirical_data.loc[ # type: ignore + :, # type: ignore + edp, + ].values, ) ) else: # all other RVs need parameters of their distributions - RV_reg.add_RV( - uq.rv_class_map(family)( + rv_reg.add_RV( + uq.rv_class_map(family)( # type: ignore name=rv_tag, - theta=[ - getattr(rv_params, f"Theta_{t_i}", np.nan) - for t_i in range(3) - ], - truncation_limits=[ - getattr(rv_params, f"Truncate{side}", np.nan) - for side in ("Lower", "Upper") - ], + theta=np.array( + [ + value + for t_i in range(3) + if ( + value := getattr(rv_params, f'Theta_{t_i}', None) + ) + is not None + ] + ), + truncation_limits=np.array( + [ + getattr(rv_params, f'Truncate{side}', np.nan) + for side in ('Lower', 'Upper') + ] + ), ) ) self.log.msg( - f"\n{self.marginal_params.shape[0]} random variables created.", + f'\n{self.marginal_params.shape[0]} random variables created.', prepend_timestamp=False, ) @@ -923,32 +1006,33 @@ def _create_RVs(self, preserve_order: bool = False) -> None: if self.correlation is not None: rv_set_tags = [ f'EDP-{edp[0]}-{edp[1]}-{edp[2]}' - for edp in self.correlation.index.values + for edp in self.correlation.index.to_numpy() ] - RV_reg.add_RV_set( + rv_reg.add_RV_set( uq.RandomVariableSet( 'EDP_set', - list(RV_reg.RVs(rv_set_tags).values()), + list(rv_reg.RVs(rv_set_tags).values()), self.correlation.values, ) ) self.log.msg( - f"\nCorrelations between {len(rv_set_tags)} random variables " - "successfully defined.", + f'\nCorrelations between {len(rv_set_tags)} random variables ' + 'successfully defined.', prepend_timestamp=False, ) - self._RVs = RV_reg + self._RVs = rv_reg def clone_demands(self, demand_cloning: dict) -> None: """ - Clones demands. This means copying over columns of the - original demand sample and assigning given names to them. The - columns to be copied over and the names to assign to the - copies are defined as the keys and values of the - `demand_cloning` dictionary, respectively. + Clone demands. + + Copies over columns of the original demand sample and + assigns given names to them. The columns to be copied over + and the names to be assigned to the copies are defined as the keys + and values of the `demand_cloning` dictionary. The method modifies `sample` inplace. Parameters @@ -969,7 +1053,6 @@ def clone_demands(self, demand_cloning: dict) -> None: In multiple instances of invalid demand_cloning entries. """ - # it's impossible to have duplicate keys, because # demand_cloning is a dictionary. new_columns_list = demand_cloning.values() @@ -984,12 +1067,11 @@ def clone_demands(self, demand_cloning: dict) -> None: for new_columns in new_columns_list: flat_list.extend(new_columns) if len(set(flat_list)) != len(flat_list): - raise ValueError('Duplicate entries in demand cloning configuration.') + msg = 'Duplicate entries in demand cloning configuration.' + raise ValueError(msg) # turn the config entries to tuples - def turn_to_tuples(demand_cloning): - # pylint: disable=missing-return-doc - # pylint: disable=missing-return-type-doc + def turn_to_tuples(demand_cloning: dict) -> dict: demand_cloning_tuples = {} for key, values in demand_cloning.items(): demand_cloning_tuples[tuple(key.split('-'))] = [ @@ -1002,15 +1084,16 @@ def turn_to_tuples(demand_cloning): # The demand cloning configuration should not include # columns that are not present in the original sample. warn_columns = [] + assert self.sample is not None for column in demand_cloning: if column not in self.sample.columns: - warn_columns.append(column) + warn_columns.append(column) # noqa: PERF401 if warn_columns: warn_columns = ['-'.join(x) for x in warn_columns] - self.log.warn( - "The demand cloning configuration lists " + self.log.warning( + 'The demand cloning configuration lists ' "columns that are not present in the original demand sample's " - f"columns: {warn_columns}." + f'columns: {warn_columns}.' ) # we iterate over the existing columns of the sample and try @@ -1035,11 +1118,14 @@ def turn_to_tuples(demand_cloning): # update the column index self.sample.columns = pd.MultiIndex.from_tuples(column_values) # update units - self.user_units = self.user_units.iloc[column_index] + self.user_units = self.user_units.iloc[column_index] # type: ignore + assert self.user_units is not None self.user_units.index = self.sample.columns def generate_sample(self, config: dict) -> None: """ + Generate the demand sample. + Generates a sample of random variables (RVs) based on the specified configuration for demand modeling. @@ -1052,7 +1138,7 @@ def generate_sample(self, config: dict) -> None: Parameters ---------- - config : dict + config: dict A dictionary containing configuration options for the sample generation. Key options include: * 'SampleSize': The number of samples to generate. @@ -1088,20 +1174,23 @@ def generate_sample(self, config: dict) -> None: >>> model.generate_sample(config) # This will generate 1000 realizations of demand variables # with the specified configuration. - """ + """ if self.marginal_params is None: - raise ValueError( + msg = ( 'Model parameters have not been specified. Either ' 'load parameters from a file or calibrate the ' 'model using raw demand data.' ) + raise ValueError(msg) self.log.div() self.log.msg('Generating sample from demand variables...') self._create_RVs(preserve_order=config.get('PreserveRawOrder', False)) + assert self._RVs is not None + assert self._asmnt.options.sampling_method is not None sample_size = config['SampleSize'] self._RVs.generate_sample( sample_size=sample_size, method=self._asmnt.options.sampling_method @@ -1111,10 +1200,12 @@ def generate_sample(self, config: dict) -> None: assert self._RVs is not None assert self._RVs.RV_sample is not None sample = pd.DataFrame(self._RVs.RV_sample) - sample.sort_index(axis=0, inplace=True) - sample.sort_index(axis=1, inplace=True) + sample = sample.sort_index(axis=0) + sample = sample.sort_index(axis=1) - sample = base.convert_to_MultiIndex(sample, axis=1)['EDP'] + sample_mi = base.convert_to_MultiIndex(sample, axis=1)['EDP'] + assert isinstance(sample_mi, pd.DataFrame) + sample = sample_mi sample.columns.names = ['type', 'loc', 'dir'] self.sample = sample @@ -1123,7 +1214,7 @@ def generate_sample(self, config: dict) -> None: self.clone_demands(config['DemandCloning']) self.log.msg( - f"\nSuccessfully generated {sample_size} realizations.", + f'\nSuccessfully generated {sample_size} realizations.', prepend_timestamp=False, ) @@ -1134,27 +1225,24 @@ def _get_required_demand_type( demand_offset: dict | None = None, ) -> dict: """ - Returns the id of the demand needed to calculate damage or - loss of a component. - - This method returns the demand type and its properties - required to calculate the the damage or loss of a - component. The properties include whether the demand is - directional, the offset, and the type of the demand. The - method takes as input a dataframe `PGB` that contains + Get the required demand type for the components. + + Returns the demand type and its properties required to calculate + the the damage or loss of a component. The properties include + whether the demand is directional, the offset, and the type of the + demand. The method takes as input a dataframe `PGB` that contains information about the component groups in the asset. For each - performance group PG in the PGB dataframe, the method - retrieves the relevant parameters from the model_params - dataframe and parses the demand type into its properties. If - the demand type has a subtype, the method splits it and adds - the subtype to the demand type to form the EDP type. The - method also considers the default offset for the demand type, - if it is specified in the options attribute of the assessment, - and adds the offset to the EDP. If the demand is directional, - the direction is added to the EDP. The method collects all the - unique EDPs for each component group and returns them as a - dictionary where each key is an EDP and its value is a list of - component groups that require that EDP. + performance group PG in the PGB dataframe, the method retrieves + the relevant parameters from the model_params dataframe and parses + the demand type into its properties. If the demand type has a + subtype, the method splits it and adds the subtype to the demand + type to form the EDP type. The method also considers the default + offset for the demand type, if it is specified in the options + attribute of the assessment, and adds the offset to the EDP. If + the demand is directional, the direction is added to the EDP. The + method collects all the unique EDPs for each component group and + returns them as a dictionary where each key is an EDP and its + value is a list of component groups that require that EDP. Parameters ---------- @@ -1177,8 +1265,13 @@ def _get_required_demand_type( corresponding value is a list of tuples (component_id, location, direction) - """ + Raises + ------ + ValueError + When a negative value is used for `loc`. Currently not + supported. + """ model_parameters = model_parameters.sort_index(axis=1) # Assign default demand_offset to empty dict. @@ -1188,30 +1281,47 @@ def _get_required_demand_type( required_edps = defaultdict(list) for pg in pgb.index: - cmp = pg[0] - # Get the directional, offset, and demand_type parameters - # from the `model_parameters` DataFrame - directional = model_parameters.at[cmp, ('Demand', 'Directional')] - offset = model_parameters.at[cmp, ('Demand', 'Offset')] - demand_type = model_parameters.at[cmp, ('Demand', 'Type')] - # Utility Demand: if there is an `Expression`, then load the # rest of the demand types. expression = model_parameters.loc[cmp, :].get(('Demand', 'Expression')) if expression is not None: - demand_types = [] - for row, value in model_parameters.loc[cmp, 'Demand'].dropna().items(): - if isinstance(row, str) and row.startswith('Type'): - demand_types.append(value) + # get the number of variables in the expression using + # the numexpr library + parsed_expr = ne.NumExpr(_clean_up_expression(expression)) + num_terms = len(parsed_expr.input_names) + demand_parameters_list = [] + for i in range(num_terms): + if i == 0: + index_lvl0 = 'Demand' + else: + index_lvl0 = f'Demand{i + 1}' + demand_parameters_list.append( + ( + model_parameters.loc[cmp, (index_lvl0, 'Type')], + model_parameters.loc[cmp, (index_lvl0, 'Offset')], + model_parameters.loc[cmp, (index_lvl0, 'Directional')], + ) + ) else: - demand_types = [demand_type] + demand_parameters_list = [ + ( + model_parameters.loc[cmp, ('Demand', 'Type')], + model_parameters.loc[cmp, ('Demand', 'Offset')], + model_parameters.loc[cmp, ('Demand', 'Directional')], + ) + ] # Parse the demand type edps = [] - for demand_type in demand_types: + for demand_parameters in demand_parameters_list: + demand_type = demand_parameters[0] + offset = demand_parameters[1] + directional = demand_parameters[2] + + assert isinstance(demand_type, str) # Check if there is a subtype included in the demand_type # string @@ -1224,47 +1334,41 @@ def _get_required_demand_type( demand_type = base.EDP_to_demand_type[demand_type] # Concatenate the demand type and subtype to form the # EDP type - EDP_type = f'{demand_type}_{subtype}' + edp_type = f'{demand_type}_{subtype}' else: # If there is no subtype, convert the demand type to # the corresponding EDP type using # `base.EDP_to_demand_type` demand_type = base.EDP_to_demand_type[demand_type] # Assign the EDP type to be equal to the demand type - EDP_type = demand_type + edp_type = demand_type # Consider the default offset, if needed - if demand_type in demand_offset.keys(): + if demand_type in demand_offset: # If the demand type has a default offset in # `demand_offset`, add the offset # to the default offset - offset = int(offset + demand_offset[demand_type]) + offset = int(offset + demand_offset[demand_type]) # type: ignore else: # If the demand type does not have a default offset in # `demand_offset`, convert the # offset to an integer - offset = int(offset) + offset = int(offset) # type: ignore # Determine the direction - if directional: - # If the demand is directional, use the third element - # of the `PG` tuple as the direction - direction = pg[2] - else: - # If the demand is not directional, use '0' as the - # direction - direction = '0' + direction = pg[2] if directional else '0' # Concatenate the EDP type, offset, and direction to form # the EDP key - edp = f"{EDP_type}-{str(int(pg[1]) + offset)}-{direction}" + edp = f'{edp_type}-{int(pg[1]) + offset!s}-{direction}' if int(pg[1]) + offset < 0: - raise ValueError( + msg = ( f'Negative location encountered for component ' f'(cmp, loc, dir, uid)=`{pg}`. Would require `{edp}`. ' f'Please update the location of the component.' ) + raise ValueError(msg) edps.append(edp) @@ -1272,7 +1376,7 @@ def _get_required_demand_type( # Add the current PG (performance group) to the list of # PGs associated with the current EDP key - required_edps[(edps_t, expression)].append(pg) + required_edps[edps_t, expression].append(pg) # Return the required EDPs return required_edps @@ -1294,7 +1398,11 @@ def _assemble_required_demand_data( Parameters ---------- required_edps: set - Set of required EDPs + Set of required EDPs. The elements in the set should be + tuples. For each, the first element should be a tuple + containing EDPs in the `type`-`loc`-`dir` format. The second + should be an expression defining how the EDPs in the tuple + should be combined when it contains more than a single EDP. nondirectional_multipliers: dict Nondirectional components are sensitive to demands coming in any direction. Results are typically available in two @@ -1312,7 +1420,7 @@ def _assemble_required_demand_data( Returns ------- - demand_dict : dict + demand_dict: dict A dictionary of assembled demand data for calculation Raises @@ -1321,22 +1429,23 @@ def _assemble_required_demand_data( If demand data for a given EDP cannot be found """ - demand_dict = {} for edps, expression in required_edps: - edp_values = {} for edp in edps: - edp_type, location, direction = edp.split('-') if direction == '0': - # non-directional demand = ( - demand_sample.loc[:, (edp_type, location)].max(axis=1).values + demand_sample.loc[ + :, # type: ignore + (edp_type, location), + ] + .max(axis=1) + .to_numpy() ) if edp_type in nondirectional_multipliers: @@ -1346,18 +1455,18 @@ def _assemble_required_demand_data( multiplier = nondirectional_multipliers['ALL'] else: - raise ValueError( - f"Peak orthogonal EDP multiplier " - f"for non-directional demand " - f"calculation of `{edp_type}` not specified." + msg = ( + f'Peak orthogonal EDP multiplier ' + f'for non-directional demand ' + f'calculation of `{edp_type}` not specified.' ) + raise ValueError(msg) - demand = demand * multiplier + demand *= multiplier else: - # directional - demand = demand_sample[(edp_type, location, direction)].values + demand = demand_sample[edp_type, location, direction].to_numpy() edp_values[edp] = demand @@ -1366,24 +1475,26 @@ def _assemble_required_demand_data( # build a dict of values value_dict = {} for i, edp_value in enumerate(edp_values.values()): - value_dict[f'X{i+1}'] = edp_value + value_dict[f'X{i + 1}'] = edp_value demand = ne.evaluate( _clean_up_expression(expression), local_dict=value_dict ) - demand_dict[(edps, expression)] = demand + demand_dict[edps, expression] = demand return demand_dict def _clean_up_expression(expression: str) -> str: """ + Clean up a mathematical expression in a string. + Cleans up the given mathematical expression by ensuring it contains only allowed characters and replaces the caret (^) exponentiation operator with the double asterisk (**) operator. Parameters ---------- - expression : str + expression: str The mathematical expression to clean up. Returns @@ -1410,18 +1521,21 @@ def _clean_up_expression(expression: str) -> str: ... "for x in i.repeat(0)]" ... ) Traceback (most recent call last): ... + """ allowed_chars = re.compile(r'^[0-9a-zA-Z\^\+\-\*/\(\)\s]*$') if not bool(allowed_chars.match(expression)): - raise ValueError(f'Invalid expression: {expression}') + msg = f'Invalid expression: {expression}' + raise ValueError(msg) # replace exponantiation with `^` with the native `**` in case `^` # was used. But please use `**`.. - expression = expression.replace('^', '**') - return expression + return expression.replace('^', '**') def _verify_edps_available(available_edps: dict, required: set) -> None: """ + Verify EDP availability. + Verifies that the required EDPs are available and raises appropriate errors otherwise. @@ -1447,19 +1561,20 @@ def _verify_edps_available(available_edps: dict, required: set) -> None: for edp in edps: edp_type, location, direction = edp.split('-') if (edp_type, location) not in available_edps: - raise ValueError( + msg = ( f'Unable to locate `{edp_type}` at location ' f'{location} in demand sample.' ) + raise ValueError(msg) # if non-directional demand is requested, ensure there # are entries (all directions accepted) - num_entries = len(available_edps[(edp_type, location)]) + num_entries = len(available_edps[edp_type, location]) if edp[2] == '0' and num_entries == 0: - raise ValueError( + msg = ( f'Unable to locate any `{edp_type}` ' f'at location {location} and direction {direction}.' ) + raise ValueError(msg) if edp[2] != '0' and num_entries == 0: - raise ValueError( - f'Unable to locate `{edp_type}-{location}-{direction}`.' - ) + msg = f'Unable to locate `{edp_type}-{location}-{direction}`.' + raise ValueError(msg) diff --git a/pelicun/model/loss_model.py b/pelicun/model/loss_model.py index 1742fc867..6c6c3be54 100644 --- a/pelicun/model/loss_model.py +++ b/pelicun/model/loss_model.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -38,34 +37,34 @@ # Adam Zsarnóczay # John Vouvakis Manousakis -""" -This file defines Loss model objects and their methods. -.. rubric:: Contents - -.. autosummary:: - - LossModel - -""" +"""Loss model objects and associated methods.""" from __future__ import annotations -from typing import TYPE_CHECKING -from itertools import product + +from abc import ABC, abstractmethod from collections import defaultdict +from itertools import product +from pathlib import Path +from typing import TYPE_CHECKING, Any, overload + import numpy as np import pandas as pd from scipy.interpolate import RegularGridInterpolator + +from pelicun import base, file_io, uq +from pelicun.model.demand_model import ( + _assemble_required_demand_data, + _get_required_demand_type, + _verify_edps_available, +) from pelicun.model.pelicun_model import PelicunModel -from pelicun.model.demand_model import _get_required_demand_type -from pelicun.model.demand_model import _assemble_required_demand_data -from pelicun.model.demand_model import _verify_edps_available -from pelicun import base -from pelicun import uq -from pelicun import file_io +from pelicun.pelicun_warnings import InconsistentUnitsError if TYPE_CHECKING: - from pelicun.assessment import Assessment + from collections.abc import Callable + + from pelicun.assessment import AssessmentBase idx = base.idx @@ -79,19 +78,20 @@ class LossModel(PelicunModel): """ - __slots__ = ['ds_model', 'lf_model'] + __slots__ = ['ds_model', 'dv_units', 'lf_model'] def __init__( self, - assessment: Assessment, - decision_variables: tuple[str, ...] = ('Carbon', 'Cost', 'Energy', 'Time'), - ): + assessment: AssessmentBase, + decision_variables: tuple[str, ...] = ('Cost', 'Time'), + dv_units: dict[str, str] | None = None, + ) -> None: """ - Initializes LossModel objects. + Initialize LossModel objects. Parameters ---------- - assessment: pelicun.Assessment + assessment: pelicun.AssessmentBase Parent assessment decision_variables: tuple Defines the decision variables to be included in the loss @@ -106,47 +106,64 @@ def __init__( self.lf_model: RepairModel_LF = RepairModel_LF(assessment) self._loss_map = None self.decision_variables = decision_variables + self.dv_units = dv_units @property - def sample(self): + def sample(self) -> pd.DataFrame | None: """ - + Combines the samples of the ds_model and lf_model sub-models. Returns ------- pd.DataFrame - The damage state-driven component loss sample. + The combined loss sample. """ - self.log.warn( - '`{loss model}.sample` is deprecated and will be dropped in ' - 'future versions of pelicun. ' - 'Please use `{loss model}.ds_model.sample` ' - 'or `{loss model}.lf_model.sample` instead. ' - 'Now returning {loss model}.ds_model.sample`.' - ) - return self.ds_model.sample + # Handle `None` cases + + if self.ds_model.sample is None and self.lf_model.sample is None: + return None + + if self.ds_model.sample is None: + return self.lf_model.sample + + if self.lf_model.sample is None: + return self.ds_model.sample + + # If both are not None, combine + + ds_model_levels = self.ds_model.sample.columns.names + + # add a `ds` level to the lf_model sample + new_index = self.lf_model.sample.columns.to_frame(index=False) + # add + new_index['ds'] = 'N/A' + # reorder + new_index = new_index[ds_model_levels] + new_multiindex = pd.MultiIndex.from_frame(new_index) + self.lf_model.sample.columns = new_multiindex + + return pd.concat((self.ds_model.sample, self.lf_model.sample), axis=1) @property - def decision_variables(self): + def decision_variables(self) -> tuple[str, ...]: """ - Retrieves the decision variables to be used in the loss - calculations. + Retrieve the decision variables. Returns ------- tuple Decision variables. + """ # pick the object from one of the models # it's the same for the other(s). return self.ds_model.decision_variables @decision_variables.setter - def decision_variables(self, decision_variables): + def decision_variables(self, decision_variables: tuple[str, ...]) -> None: """ - Sets the decision variables to be used in the loss - calculations. + Set the decision variables. Supported: {`Cost`, `Time`, `Energy`, `Carbon`}. Could also be any other string, as long as the provided loss @@ -163,9 +180,10 @@ def add_loss_map( loss_map_policy: str | None = None, ) -> None: """ - Add a loss map to the loss model. A loss map defines what loss - parameter definition should be used for each component ID in - the asset model. + Add a loss map to the loss model. + + A loss map defines what loss parameter definition should be + used for each component ID in the asset model. Parameters ---------- @@ -188,18 +206,16 @@ def add_loss_map( If both arguments are None. """ - self.log.msg('Loading loss map...') # If no loss map is provided and no default is requested, # there is no loss map and we can't proceed. if loss_map_path is None and loss_map_policy is None: - raise ValueError( - 'Please provide a loss map and/or a loss map extension policy.' - ) + msg = 'Please provide a loss map and/or a loss map extension policy.' + raise ValueError(msg) # get a list of unique component IDs - cmp_set = self._asmnt.asset.list_unique_component_ids(as_set=True) + cmp_set = set(self._asmnt.asset.list_unique_component_ids()) if loss_map_path is not None: self.log.msg('Loss map is provided.', prepend_timestamp=False) @@ -211,9 +227,10 @@ def add_loss_map( reindex=False, log=self._asmnt.log, ) + assert isinstance(loss_map, pd.DataFrame) # - if np.any(['DMG' in x for x in loss_map.index]): - self.log.warn( + if np.any(['DMG' in x for x in loss_map.index]): # type: ignore + self.log.warning( 'The `DMG-` flag in the loss_map index is deprecated ' 'and no longer necessary. ' 'Please do not prepend `DMG-` before the component ' @@ -239,9 +256,10 @@ def add_loss_map( # Don't do anything. pass - # TODO: add other loss map policies. + # TODO(AZ): add other loss map policies. else: - raise ValueError(f'Unknown loss map policy: `{loss_map_policy}`.') + msg = f'Unknown loss map policy: `{loss_map_policy}`.' + raise ValueError(msg) # Assign the loss map to the available loss models self._loss_map = loss_map @@ -254,11 +272,8 @@ def load_model( loss_map: str | pd.DataFrame, decision_variables: tuple[str, ...] | None = None, ) -> None: - """ - - - """ - self.log.warn( + """.""" + self.log.warning( '`load_model` is deprecated and will be dropped in ' 'future versions of pelicun. ' 'Please use `load_model_parameters` instead.' @@ -283,23 +298,21 @@ def load_model_parameters( prior elements in the list take precedence over the same parameters in subsequent data paths. I.e., place the Default datasets in the back. - - Raises - ------ - ValueError - If the method can't parse the loss parameters in the - specified paths. + decision_variables: tuple + Defines the decision variables to be included in the loss + calculations. Defaults to those supported, but fewer can be + used if desired. When fewer are used, the loss parameters for + those not used will not be required. """ - if decision_variables is not None: # - self.decision_variables = set(decision_variables) - self.log.warn( + self.decision_variables = decision_variables + self.log.warning( 'The `decision_variables` argument has been removed. ' 'Please set your desired decision variables like so: ' '{assessment object}.loss.decision_variables ' - '= (\'dv1\', \'dv2\', ...) before calling ' + "= ('dv1', 'dv2', ...) before calling " '{assessment object}.add_loss_map().' ) @@ -314,24 +327,7 @@ def load_model_parameters( # for data_path in data_paths: - if 'bldg_repair_DB' in data_path: - data_path = data_path.replace('bldg_repair_DB', 'loss_repair_DB') - self.log.warn( - '`bldg_repair_DB` is deprecated and will ' - 'be dropped in future versions of pelicun. ' - 'Please use `loss_repair_DB` instead.' - ) - data = file_io.load_data( - data_path, None, orientation=1, reindex=False, log=self._asmnt.log - ) - # determine if the loss model parameters are for damage - # states or loss functions - if _is_for_ds_model(data): - self.ds_model._load_model_parameters(data) - elif _is_for_lf_model(data): - self.lf_model._load_model_parameters(data) - else: - raise ValueError(f'Invalid loss model parameters: {data_path}') + self._load_from_data_path(data_path) self.log.msg( 'Loss model parameters loaded successfully.', prepend_timestamp=False @@ -345,14 +341,35 @@ def load_model_parameters( 'Removing unused loss model parameters.', prepend_timestamp=False ) + assert self._loss_map is not None for loss_model in self._loss_models: # drop unused loss parameter definitions - loss_model._drop_unused_loss_parameters(self._loss_map) + loss_model.drop_unused_loss_parameters(self._loss_map) # remove components with incomplete loss parameters - loss_model._remove_incomplete_components() + loss_model.remove_incomplete_components() # drop unused damage state columns - self.ds_model._drop_unused_damage_states() + self.ds_model.drop_unused_damage_states() + + # + # obtain DV units + # + dv_units: dict = {} + if self.ds_model.loss_params is not None: + dv_units.update( + self.ds_model.loss_params['DV', 'Unit'] + .groupby(level=[1]) + .first() + .to_dict() + ) + if self.lf_model.loss_params is not None: + dv_units.update( + self.lf_model.loss_params['DV', 'Unit'] + .groupby(level=[1]) + .first() + .to_dict() + ) + self.dv_units = dv_units # # convert units @@ -362,7 +379,7 @@ def load_model_parameters( 'Converting loss model parameter units.', prepend_timestamp=False ) for loss_model in self._loss_models: - loss_model._convert_loss_parameter_units() + loss_model.convert_loss_parameter_units() # # verify loss parameter availability @@ -375,6 +392,39 @@ def load_model_parameters( ) self._ensure_loss_parameter_availability() + def _load_from_data_path(self, data_path: str | pd.DataFrame) -> None: + if 'bldg_repair_DB' in data_path: + data_path = data_path.replace('bldg_repair_DB', 'loss_repair_DB') + self.log.warning( + '`bldg_repair_DB` is deprecated and will ' + 'be dropped in future versions of pelicun. ' + 'Please use `loss_repair_DB` instead.' + ) + data = file_io.load_data( + data_path, None, orientation=1, reindex=False, log=self._asmnt.log + ) + assert isinstance(data, pd.DataFrame) + + # Check for unit consistency + data.index.names = ['cmp', 'dv'] + units_isolated = data.reset_index()[[('dv', ''), ('DV', 'Unit')]] + units_isolated.columns = pd.Index(['dv', 'Units']) + units_isolated_grp = units_isolated.groupby('dv')['Units'] + unit_counts = units_isolated_grp.nunique() + more_than_one = unit_counts[unit_counts > 1] + if not more_than_one.empty: + raise InconsistentUnitsError + + # determine if the loss model parameters are for damage + # states or loss functions + if _is_for_ds_model(data): + self.ds_model.load_model_parameters(data) + elif _is_for_lf_model(data): + self.lf_model.load_model_parameters(data) + else: + msg = f'Invalid loss model parameters: {data_path}' + raise ValueError(msg) + def calculate(self) -> None: """ Calculate the loss of each component block. @@ -396,24 +446,30 @@ def calculate(self) -> None: # Get the damaged quantities in each damage state for each # component of interest. - # TODO: FIND A WAY to avoid making a copy of this. demand = self._asmnt.demand.sample + assert demand is not None demand_offset = self._asmnt.options.demand_offset + assert demand_offset is not None nondirectional_multipliers = self._asmnt.options.nondir_multi_dict + assert nondirectional_multipliers is not None + assert self._asmnt.asset.cmp_sample is not None cmp_sample = self._asmnt.asset.cmp_sample.to_dict('series') cmp_marginal_params = self._asmnt.asset.cmp_marginal_params + assert cmp_marginal_params is not None if self._asmnt.damage.ds_model.sample is not None: + # TODO(JVM): FIND A WAY to avoid making a copy of this. dmg_quantities = self._asmnt.damage.ds_model.sample.copy() if len(demand) != len(dmg_quantities): - raise ValueError( + msg = ( f'The demand sample contains {len(demand)} realizations, ' f'but the damage sample contains {len(dmg_quantities)}. ' f'Loss calculation cannot proceed when ' f'these numbers are different. ' ) - self.ds_model._calculate(dmg_quantities) + raise ValueError(msg) + self.ds_model.calculate(dmg_quantities) - self.lf_model._calculate( + self.lf_model.calculate( demand, cmp_sample, cmp_marginal_params, @@ -421,31 +477,31 @@ def calculate(self) -> None: nondirectional_multipliers, ) - self.log.msg("Loss calculation successful.") + self.log.msg('Loss calculation successful.') def consequence_scaling(self, scaling_specification: str) -> None: """ + Apply scale factors to losses. + Applies scale factors to the loss sample according to the - given scaling specification. - - The scaling specification should be a path to a CSV file. It - should contain a `Decision Variable` column with a specified - decision variable in each row. Other optional columns are - `Component`, `Location`, `Direction`. Each row acts as an - independent scaling operation, with the scale factor defined - in the `Scale Factor` column, which is required. If any - value is missing in the optional columns, it is assumed that - the scale factor should be applied to all entries of the - loss sample where the other column values match. For example, - if the specification has a single row with `Decision Variable` - set to 'Cost', `Scale Factor` set to 2.0, and no other - columns, this will double the 'Cost' DV. If instead `Location` - was also set to `1`, it would double the Cost of all - components that have that location. The columns `Location` and - `Direction` can contain ranges, like this: `1--3` means - `1`, `2`, and `3`. If a range is used in both `Location` and - `Direction`, the factor of that row will be applied once to - all combinations. + given scaling specification. The scaling specification should + be a path to a CSV file. It should contain a `Decision + Variable` column with a specified decision variable in each + row. Other optional columns are `Component`, `Location`, + `Direction`. Each row acts as an independent scaling + operation, with the scale factor defined in the `Scale Factor` + column, which is required. If any value is missing in the + optional columns, it is assumed that the scale factor should + be applied to all entries of the loss sample where the other + column values match. For example, if the specification has a + single row with `Decision Variable` set to 'Cost', `Scale + Factor` set to 2.0, and no other columns, this will double the + 'Cost' DV. If instead `Location` was also set to `1`, it would + double the Cost of all components that have that location. The + columns `Location` and `Direction` can contain ranges, like + this: `1--3` means `1`, `2`, and `3`. If a range is used in + both `Location` and `Direction`, the factor of that row will + be applied once to all combinations. Parameters ---------- @@ -458,7 +514,6 @@ def consequence_scaling(self, scaling_specification: str) -> None: If required columns are missing or contain NaNs. """ - # Specify expected dtypes from the start. dtypes = { 'Decision Variable': 'str', @@ -474,18 +529,20 @@ def consequence_scaling(self, scaling_specification: str) -> None: 'Decision Variable' not in scaling_specification_df.columns or scaling_specification_df['Decision Variable'].isna().any() ): - raise ValueError( + msg = ( 'The `Decision Variable` column is missing ' 'from the scaling specification or contains NaN values.' ) + raise ValueError(msg) if ( 'Scale Factor' not in scaling_specification_df.columns or scaling_specification_df['Scale Factor'].isna().any() ): - raise ValueError( + msg = ( 'The `Scale Factor` column is missing ' 'from the scaling specification or contains NaN values.' ) + raise ValueError(msg) # Add missing optional columns with NaN values optional_cols = ['Component', 'Location', 'Direction'] @@ -501,19 +558,19 @@ def consequence_scaling(self, scaling_specification: str) -> None: 'Direction': 'dir', 'Scale Factor': 'scaling', } - scaling_specification_df.rename(columns=name_map, inplace=True) + scaling_specification_df = scaling_specification_df.rename(columns=name_map) # Expand ranges in 'loc' and 'dir' - def _expand_range(col): + def _expand_range(col): # noqa: ANN001, ANN202 if pd.isna(col): return [col] if '--' in col: - start, end = [int(x) for x in col.split('--')] + start, end = (int(x) for x in col.split('--')) return [str(x) for x in range(start, end + 1)] return [col] # Generate all combinations of loc and dir ranges - expanded_df = scaling_specification_df.apply( + expanded_df = scaling_specification_df.apply( # type: ignore lambda row: pd.DataFrame( list(product(_expand_range(row['loc']), _expand_range(row['dir']))), columns=['loc', 'dir'], @@ -537,11 +594,11 @@ def _apply_consequence_scaling( self, scaling_conditions: dict, scale_factor: float, + *, raise_missing: bool = True, ) -> None: """ - Applies a scale factor to selected columns of the loss - samples. + Apply a scale factor to selected loss sample columns. The scaling conditions are passed as a dictionary mapping level names with their required value for the condition to be @@ -567,6 +624,8 @@ def _apply_consequence_scaling( which case only the matching rows will be affected. scale_factor: float Scale factor to use. + raise_missing: bool + Raise an error if no rows are matching the given conditions. Raises ------ @@ -575,18 +634,17 @@ def _apply_consequence_scaling( `dv` key. """ - # make sure we won't apply the same factor to all DVs at once, # highly unlikely anyone would actually want to do this. if 'dv' not in scaling_conditions: - raise ValueError( + msg = ( 'The index of the `scaling_conditions` dictionary ' 'should contain a level named `dv` listing the ' 'relevant decision variable.' ) + raise ValueError(msg) for model in self._loss_models: - # check if it's empty if model.sample is None: continue @@ -595,9 +653,10 @@ def _apply_consequence_scaling( # values exist yet) for name in scaling_conditions: if name not in model.sample.columns.names: - raise ValueError( + msg = ( f'`scaling_conditions` contains an unknown level: `{name}`.' ) + raise ValueError(msg) # apply scale factors base.multiply_factor_multiple_levels( @@ -609,10 +668,10 @@ def _apply_consequence_scaling( ) def save_sample( - self, filepath: str | None = None, save_units: bool = False - ) -> None | tuple[pd.DataFrame, pd.Series]: + self, filepath: str | None = None, *, save_units: bool = False + ) -> None | pd.DataFrame | tuple[pd.DataFrame, pd.Series]: """ - + . Saves the sample of the `ds_model`. @@ -622,7 +681,7 @@ def save_sample( The output of {loss model}.ds_model.save_sample. """ - self.log.warn( + self.log.warning( '`{loss model}.save_sample` is deprecated and will raise ' 'in future versions of pelicun. Please use ' '{loss model}.ds_model.save_sample instead.' @@ -631,28 +690,30 @@ def save_sample( def load_sample(self, filepath: str | pd.DataFrame) -> None: """ - + . Saves the sample of the `ds_model`. """ - self.log.warn( + self.log.warning( '`{loss model}.load_sample` is deprecated and will raise ' 'in future versions of pelicun. Please use ' '{loss model}.ds_model.load_sample instead.' ) - self.ds_model.load_sample(filepath=filepath) + dv_units = self.ds_model.load_sample(filepath=filepath) + self.dv_units = dv_units - def aggregate_losses( + def aggregate_losses( # noqa: C901 self, replacement_configuration: ( tuple[uq.RandomVariableRegistry, dict[str, float]] | None ) = None, loss_combination: dict | None = None, + *, future: bool = False, ) -> pd.DataFrame | tuple[pd.DataFrame, pd.DataFrame]: """ - Aggregates the losses produced by each component. + Aggregate the losses produced by each component. Parameters ---------- @@ -664,7 +725,7 @@ def aggregate_losses( thresholds. If the aggregated value for a decision variable (conditioned on no replacement) exceeds the threshold, then replacement is triggered. This can happen - for multuple decision variables at the same + for multiple decision variables at the same realization. The consequence keyword `replacement` is reserved to represent exclusive triggering of the replacement consequences, and other consequences are @@ -672,9 +733,9 @@ def aggregate_losses( triggered. When assigned to None, then `replacement` is still treated as an exclusive consequence (other consequences are set to zero when replacement is nonzero) - but it is not being additinally triggered by the + but it is not being additionally triggered by the exceedance of any thresholds. The aggregated loss sample - conains an additional column with information on whether + contains an additional column with information on whether replacement was already present or triggered by a threshold exceedance for each realization. loss_combination: dict, optional @@ -698,33 +759,30 @@ def aggregate_losses( This structure allows for the loss combination of M components. In this case the (`c1`, `c2`) tuple should contain M elements instead of two. + future: bool, optional + Defaults to False. When set to True, it enables the + updated return type. - Note - ---- + Notes + ----- Regardless of the value of the arguments, this method does not alter the state of the loss model, i.e., it does not modify the values of the `.sample` attributes. Returns ------- - tuple + dataframe or tuple Dataframe with the aggregated loss of each realization, and another boolean dataframe with information on which DV thresholds were exceeded in each realization, triggering replacement. If no thresholds are specified it only - contains False values. - - Raises - ------ - ValueError - When inputs are invalid. + contains False values. The second dataframe is only + returned with `future` set to True. """ - - # TODO - # When we start working on the documentation, simplify the - # docstring above and point the relevant detailed section in - # the documentation. + # TODO(JVM): When we start working on the documentation, + # simplify the docstring above and point the relevant detailed + # section in the documentation. # validate input if replacement_configuration is not None: @@ -746,7 +804,7 @@ def aggregate_losses( else: lf_sample = None - def _construct_columns(): + def _construct_columns() -> list[str]: columns = [ f'repair_{x.lower()}' for x in self.decision_variables if x != 'Time' ] @@ -757,9 +815,8 @@ def _construct_columns(): return columns if ds_sample is None and lf_sample is None: - self.log.msg("There are no losses.") - df_agg = pd.DataFrame(0.00, index=[0], columns=_construct_columns()) - return df_agg + self.log.msg('There are no losses.') + return pd.DataFrame(0.00, index=[0], columns=_construct_columns()) # # handle `replacement`, regardless of whether @@ -776,13 +833,12 @@ def _construct_columns(): # levels to preserve (this aggregates `ds` for the ds_model) column_levels = ['dv', 'loss', 'dmg', 'loc', 'dir', 'uid'] - samples = [ - sample.groupby(by=column_levels, axis=1).sum() - for sample in (ds_sample, lf_sample) - if sample is not None - ] - sample = pd.concat(samples, axis=1) - sample = sample.sort_index(axis=1) + combined_sample = self.sample + sample = ( + combined_sample.groupby(by=column_levels, axis=1) # type: ignore + .sum() + .sort_index(axis=1) + ) # # perform loss combinations (special non-additive @@ -804,7 +860,7 @@ def _construct_columns(): df_agg = self._aggregate_sample(sample, _construct_columns()) if not future: - self.log.warn( + self.log.warning( '`aggregate_losses` has been expanded to support the ' 'consideration of the exceedance of loss threshold ' 'values leading to asset replacement ' @@ -823,28 +879,32 @@ def _construct_columns(): def _validate_input_loss_combination(self, loss_combination: dict) -> None: for dv, combinations in loss_combination.items(): if dv not in self.decision_variables: - raise ValueError( + msg = ( f'`loss_combination` contains the key ' f'`{dv}` which is not found in the active ' f'decision variables. These are: ' f'{self.decision_variables}.' ) + raise ValueError(msg) for components, array in combinations.items(): if not isinstance(components, tuple): - raise ValueError( + msg = ( f'Invalid type for components in loss combination ' f'for `{dv}`: {type(components)}. It should be a tuple.' ) + raise TypeError(msg) if not all(isinstance(c, str) for c in components): - raise ValueError( + msg = ( f'All elements of the components tuple in loss ' f'combination for `{dv}` should be strings.' ) + raise ValueError(msg) if not isinstance(array, np.ndarray): - raise ValueError( + msg = ( f'Invalid type for array in loss combination ' f'for `{dv}`: {type(array)}. It should be a numpy array.' ) + raise TypeError(msg) def _validate_input_replacement_thresholds( self, @@ -852,47 +912,57 @@ def _validate_input_replacement_thresholds( uq.RandomVariableRegistry, dict[str, float] ], ) -> None: - replacement_consequence_RV_reg, replacement_ratios = ( + replacement_consequence_rv_reg, replacement_ratios = ( replacement_configuration ) - if not isinstance(replacement_consequence_RV_reg, uq.RandomVariableRegistry): - raise TypeError( + if not isinstance(replacement_consequence_rv_reg, uq.RandomVariableRegistry): + msg = ( f'Invalid type for replacement consequence RV registry: ' - f'{type(replacement_consequence_RV_reg)}. It should be ' + f'{type(replacement_consequence_rv_reg)}. It should be ' f'uq.RandomVariableRegistry.' ) - for key in replacement_consequence_RV_reg.RV: + raise TypeError(msg) + for key in replacement_consequence_rv_reg.RV: if key not in self.decision_variables: - raise ValueError( + msg = ( f'`replacement_consequence_RV_reg` contains the key ' f'`{key}` which is not found in the active ' f'decision variables. These are: ' f'{self.decision_variables}.' ) + if self.query_error_setup( + 'Loss/ReplacementThreshold/RaiseOnUnknownKeys' + ): + raise ValueError(msg) + self.log.warning(msg) + for key in replacement_ratios: if key not in self.decision_variables: - raise ValueError( + msg = ( f'`replacement_ratios` contains the key ' f'`{key}` which is not found in the active ' f'decision variables. These are: ' f'{self.decision_variables}.' ) + if self.query_error_setup( + 'Loss/ReplacementThreshold/RaiseOnUnknownKeys' + ): + raise ValueError(msg) + self.log.warning(msg) # The replacement_consequence_RV_reg should contain an RV for # all active DVs, regardless of whether there is a replacement - # threshold for that DV, becauase when replacememnt is + # threshold for that DV, because when replacememnt is # triggered, we need to assign a consequence for all DVs. for key in self.decision_variables: - if key not in replacement_consequence_RV_reg.RV: - raise ValueError( - f'Missing replacement consequence RV ' f'for `{key}`.' - ) + if key not in replacement_consequence_rv_reg.RV: + msg = f'Missing replacement consequence RV ' f'for `{key}`.' + raise ValueError(msg) def _apply_loss_combinations( self, loss_combination: dict, sample: pd.DataFrame ) -> pd.DataFrame: """ - Performs non-additive loss combinations of specified - components. + Perform loss combinations of specified components. This function deconstructs the loss combination arrays, identifies the combinable components, and applies the @@ -902,14 +972,14 @@ def _apply_loss_combinations( Parameters ---------- - loss_combination : dict + loss_combination: dict A dictionary containing the loss combination information. The structure is nested dictionaries where the outer keys are decision variables, inner keys are components to combine, and the values are array objects representing the combination data. - sample : pandas.DataFrame + sample: pandas.DataFrame The input DataFrame containing the sample data. The columns are assumed to be a MultiIndex with at least the levels (decision_variable, loss_id, component_id, @@ -921,7 +991,6 @@ def _apply_loss_combinations( A new DataFrame with the combined loss data. """ - # deconstruct combination arrays to extract the input domains loss_combination_converted = self._deconstruct_loss_combination_arrays( loss_combination @@ -936,7 +1005,7 @@ def _apply_loss_combinations( # dictionary. Will be turned into a dataframe in the end. # This avoids manipulating the original sample dataframe which # would be slow. - dcsample = {} + dcsample: dict = {} # add columns to the new sample dictionary. # those that should be combined @@ -944,18 +1013,17 @@ def _apply_loss_combinations( dsample, loss_combination_converted, dcsample ) # and the remaining - for col in dsample: - dcsample[col] = dsample[col] + for col, val in dsample.items(): + dcsample[col] = val # noqa: PERF403 # turn into a dataframe - sample = pd.DataFrame(dcsample).rename_axis(columns=sample.columns.names) - return sample + return pd.DataFrame(dcsample).rename_axis(columns=sample.columns.names) def _loss_combination_add_combinable( self, dsample: dict, loss_combination_converted: dict, dcsample: dict ) -> None: """ - Adds combinable loss data. + Add combinable loss data. This function identifies groups of `loc`-`dir`-`uid` that can be combined for each decision variable and computes the @@ -966,20 +1034,20 @@ def _loss_combination_add_combinable( Parameters ---------- - dsample : dict + dsample: dict A dictionary representing the loss sample data, where keys are tuples of the form (decision_variable, loss_id, component_id, location, direction, uid) and values are the corresponding data arrays. - loss_combination_converted : dict + loss_combination_converted: dict A dictionary containing loss combination data. The structure is nested dictionaries where the outer keys are decision variables, inner keys are components to combine, and the values are tuples of combination parameters (domains and reference values). - dcsample : dict + dcsample: dict A dictionary to store the combined loss data, where keys are tuples of the form (decision_variable, 'combination', combined_component_string, location, direction, uid) and @@ -995,7 +1063,7 @@ def _loss_combination_add_combinable( # cache already defined interpolation functions. This obviates # the need to define all of them and we can just define them # on the spot when needed, and reuse them if available. - interpolation_function_cache = {} + interpolation_function_cache: dict = {} for ( decision_variable, @@ -1007,13 +1075,14 @@ def _loss_combination_add_combinable( ) in combination_data.items(): # determine if the components to combine are part of # an available group - target_group = None + target_group: dict | None = None for available_group in potential_groups[decision_variable]: # check if `components_to_combine` is a subset of # that available group if frozenset(components_to_combine) <= available_group: target_group = available_group break + assert target_group is not None # construct relevant loss sample columns for loc_dir_uid in potential_groups[decision_variable][target_group]: cols = [ @@ -1034,6 +1103,7 @@ def _loss_combination_add_combinable( interp_func = RegularGridInterpolator( domains, reference_values ) + assert interp_func is not None combined_loss = interp_func(values) combined_loss_col = ( decision_variable, @@ -1045,10 +1115,9 @@ def _loss_combination_add_combinable( for col in cols: dsample.pop(col) - def _identify_potential_groups(self, dsample: dict) -> dict: + def _identify_potential_groups(self, dsample: dict) -> dict: # noqa: PLR6301 """ - Identifies potential groups of `loc`-`dir`-`uid` for each - decision variable. + Identify potential groups of `loc`-`dir`-`uid` for each DV. This function identifies all combinations of `loc`-`dir`-`uid` that can be grouped for each decision variable based on the @@ -1056,7 +1125,7 @@ def _identify_potential_groups(self, dsample: dict) -> dict: Parameters ---------- - dsample : iterable + dsample: iterable An iterable where each containing tuple contains information about the components and their attributes. The expected format of each tuple is (decision_variable, @@ -1071,21 +1140,21 @@ def _identify_potential_groups(self, dsample: dict) -> dict: direction, uid) tuples. """ - grouped = defaultdict(defaultdict(list).copy) + grouped: defaultdict = defaultdict(defaultdict(list).copy) for col in dsample: c_dv, _, c_dmg, c_loc, c_dir, c_uid = col grouped[c_dv][c_loc, c_dir, c_uid].append(c_dmg) # invert so that we have component sets mapped to # `loc`-`dir`-`uid`s. - inverted = defaultdict(defaultdict(list).copy) + inverted: defaultdict = defaultdict(defaultdict(list).copy) for c_dv in grouped: for loc_dir_uid, component_set in grouped[c_dv].items(): inverted[c_dv][frozenset(component_set)].append(loc_dir_uid) return inverted - def _map_component_ids_to_loss_ids(self, dsample: dict) -> dict: + def _map_component_ids_to_loss_ids(self, dsample: dict) -> dict: # noqa: PLR6301 """ - Maps component IDs to loss IDs. + Map component IDs to loss IDs. This function maps components to losses based on the loss sample's columns. It assumes that multiple component IDs can @@ -1094,7 +1163,7 @@ def _map_component_ids_to_loss_ids(self, dsample: dict) -> dict: Parameters ---------- - dsample : tuple dictionary keys + dsample: tuple dictionary keys Each tuple contains information about the components and corresponding losses. @@ -1112,7 +1181,7 @@ def _map_component_ids_to_loss_ids(self, dsample: dict) -> dict: dmg_to_loss[c_dmg] = c_loss return dmg_to_loss - def _deconstruct_loss_combination_arrays(self, loss_combination: dict) -> dict: + def _deconstruct_loss_combination_arrays(self, loss_combination: dict) -> dict: # noqa: PLR6301 """ Deconstruct loss combination arrays. @@ -1123,7 +1192,7 @@ def _deconstruct_loss_combination_arrays(self, loss_combination: dict) -> dict: Parameters ---------- - loss_combination : dict + loss_combination: dict A dictionary where keys are decision variables and values are another dictionary. The inner dictionary has keys as components to combine and values as numpy array @@ -1139,7 +1208,7 @@ def _deconstruct_loss_combination_arrays(self, loss_combination: dict) -> dict: array itself. """ - loss_combination_converted = {} + loss_combination_converted: dict = {} for decision_variable, combination_data in loss_combination.items(): loss_combination_converted[decision_variable] = {} for ( @@ -1160,58 +1229,45 @@ def _deconstruct_loss_combination_arrays(self, loss_combination: dict) -> dict: def _aggregate_sample(self, sample: pd.DataFrame, columns: list) -> pd.DataFrame: """ - Sums up component losses. + Sum up component losses. + + Returns + ------- + pd.DataFrame + Dataframe with the aggregated losses. """ df_agg = pd.DataFrame(index=sample.index, columns=columns) # group results by DV type and location - aggregated = sample.groupby(level=['dv', 'loc'], axis=1).sum() + aggregated = sample.groupby( + level=['dv', 'loc'], + axis=1, # type: ignore + ).sum() for decision_variable in self.decision_variables: - # Time - if decision_variable == 'Time' and 'Time' in aggregated.columns: + if ( + decision_variable == 'Time' + and 'Time' in aggregated.columns.get_level_values('dv') + ): df_agg['repair_time-sequential'] = aggregated['Time'].sum(axis=1) df_agg['repair_time-parallel'] = aggregated['Time'].max(axis=1) - elif decision_variable == 'Time' and 'Time' not in aggregated.columns: + elif ( + decision_variable == 'Time' + and 'Time' not in aggregated.columns.get_level_values('dv') + ): df_agg = df_agg.drop( ['repair_time-parallel', 'repair_time-sequential'], axis=1 ) # All other - elif decision_variable in aggregated.columns: + elif decision_variable in aggregated.columns.get_level_values('dv'): df_agg[f'repair_{decision_variable.lower()}'] = aggregated[ decision_variable ].sum(axis=1) else: df_agg = df_agg.drop(f'repair_{decision_variable.lower()}', axis=1) - cmp_units = {} - if self.ds_model.loss_params is not None: - cmp_units.update( - self.ds_model.loss_params[('DV', 'Unit')] - .groupby(level=[1]) - .agg(lambda x: x.value_counts().index[0]) - .to_dict() - ) - if self.lf_model.loss_params is not None: - cmp_units.update( - self.lf_model.loss_params[('DV', 'Unit')] - .groupby(level=[1]) - .agg(lambda x: x.value_counts().index[0]) - .to_dict() - ) - # If the samples have been loaded to the loss model without - # loading loss parameter data, there will be no units here. - # In this case we assume default units. - if not cmp_units: - cmp_units = { - 'Cost': 'USD_2011', - 'Time': 'worker_day', - 'Carbon': 'kg', - 'Energy': 'MJ', - } - # Convert units .. column_measures = [ x.replace('repair_', '') @@ -1219,9 +1275,10 @@ def _aggregate_sample(self, sample: pd.DataFrame, columns: list) -> pd.DataFrame .replace('-parallel', '') for x in df_agg.columns.get_level_values(0) ] - column_units = [cmp_units[x.title()] for x in column_measures] + assert self.dv_units is not None + column_units = [self.dv_units[x.title()] for x in column_measures] dv_units = pd.Series(column_units, index=df_agg.columns, name='Units') - df_agg = file_io.save_to_csv( + res = file_io.save_to_csv( df_agg, None, units=dv_units, @@ -1229,66 +1286,72 @@ def _aggregate_sample(self, sample: pd.DataFrame, columns: list) -> pd.DataFrame use_simpleindex=False, log=self._asmnt.log, ) - df_agg.drop("Units", inplace=True) + assert isinstance(res, pd.DataFrame) + df_agg = res + df_agg = df_agg.drop('Units') df_agg = df_agg.astype(float) - # ouch.. - df_agg = base.convert_to_MultiIndex(df_agg, axis=1) - df_agg.sort_index(axis=1, inplace=True) + df_agg_mi = base.convert_to_MultiIndex(df_agg, axis=1) + assert isinstance(df_agg_mi, pd.DataFrame) + df_agg = df_agg_mi + df_agg = df_agg.sort_index(axis=1) df_agg = df_agg.reset_index(drop=True) + assert isinstance(df_agg, pd.DataFrame) return df_agg - def _apply_replacement_thresholds( + def _apply_replacement_thresholds( # noqa: PLR6301 self, sample: pd.DataFrame, replacement_configuration: ( tuple[uq.RandomVariableRegistry, dict[str, float]] | None ), ) -> tuple[pd.DataFrame, pd.DataFrame]: - # If there is no `replacement_configuration`, simply return. if replacement_configuration is None: # `exceedance_bool_df` is empty in this case. exceedance_bool_df = pd.DataFrame(index=sample.index, dtype=bool) return sample, exceedance_bool_df - replacement_consequence_RV_reg, replacement_ratios = ( + replacement_consequence_rv_reg, replacement_ratios = ( replacement_configuration ) - exceedance_bool_df = pd.DataFrame( - False, + exceedance_bool_df = pd.DataFrame( # type: ignore + data=False, index=sample.index, - columns=replacement_consequence_RV_reg.RV.keys(), + columns=replacement_consequence_rv_reg.RV.keys(), dtype=bool, ) # Sample replacement consequences from the registry - replacement_consequence_RV_reg.generate_sample(len(sample), 'MonteCarlo') + replacement_consequence_rv_reg.generate_sample(len(sample), 'MonteCarlo') - sample_dvs = replacement_consequence_RV_reg.RV.keys() + sample_dvs = replacement_consequence_rv_reg.RV.keys() for sample_dv in sample_dvs: sub_sample = sample.loc[:, sample_dv] if 'replacement' in sub_sample.columns.get_level_values('loss'): # If `replacement` already exists as a consequence, # determine the realizations where it is non-zero. no_replacement_mask = ( - ~(sub_sample['replacement'] > 0.00).any(axis=1).values + ~(sub_sample['replacement'] > 0.00).any(axis=1).to_numpy() ) no_replacement_columns = ( sub_sample.columns.get_level_values('loss') != 'replacement' ) else: # Otherwise there is no row where we already have replacement - no_replacement_mask = np.full(len(sub_sample), True) - no_replacement_columns = np.full(len(sub_sample.columns), True) + no_replacement_mask = np.full(len(sub_sample), fill_value=True) + no_replacement_columns = np.full( + len(sub_sample.columns), fill_value=True + ) # Get the sum to compare with the thresholds - consequence_sum_given_no_replacement = sub_sample.iloc[ + consequence_sum_given_no_replacement = sub_sample.iloc[ # type: ignore no_replacement_mask, no_replacement_columns ].sum(axis=1) if not consequence_sum_given_no_replacement.empty: - consequence_values = replacement_consequence_RV_reg.RV[ + consequence_values = replacement_consequence_rv_reg.RV[ sample_dv ].sample + assert consequence_values is not None exceedance_mask = ( consequence_sum_given_no_replacement > consequence_values[no_replacement_mask] @@ -1301,7 +1364,8 @@ def _apply_replacement_thresholds( ) else: exceedance_mask = pd.Series( - np.full(len(sub_sample), False), index=sub_sample.index + np.full(len(sub_sample), fill_value=False), + index=sub_sample.index, ) # Monitor triggering of replacement @@ -1311,7 +1375,7 @@ def _apply_replacement_thresholds( # exceeded. exceedance_realizations = exceedance_bool_df.any(axis=1) # Assign replacement consequences: needs to include all DVs - for other_dv in replacement_consequence_RV_reg.RV.keys(): + for other_dv in replacement_consequence_rv_reg.RV: col = ( other_dv, 'replacement', @@ -1325,12 +1389,13 @@ def _apply_replacement_thresholds( sample[col] = 0.00 sample = sample.sort_index(axis=1) # Assign replacement consequences - sample.loc[exceedance_realizations, col] = ( - replacement_consequence_RV_reg.RV[other_dv].sample[ - exceedance_realizations - ] - ) - # Remove all other realized consequences + other_sample = replacement_consequence_rv_reg.RV[other_dv].sample + assert other_sample is not None + sample.loc[exceedance_realizations, col] = other_sample[ + exceedance_realizations + ] + # Remove all other realized consequences from the realizations + # where the threshold was exceeded. sample.loc[ exceedance_realizations, sample.columns.get_level_values('dmg') != 'threshold_exceedance', @@ -1338,21 +1403,21 @@ def _apply_replacement_thresholds( return sample, exceedance_bool_df - def _make_replacement_exclusive( + def _make_replacement_exclusive( # noqa: PLR6301 self, ds_sample: pd.DataFrame, lf_sample: pd.DataFrame | None ) -> None: """ + Make the replacement consequence exclusive. + If `replacement` columns exist in `ds_sample`, this method treats all nonzero loss values driven by `replacement` as exclusive and zeroes-out the loss values of all other columns for the applicable rows. """ - - # columns that correspond to the replacement consequence - replacement_columns = [] # rows where replacement is non-zero - replacement_rows = [] + replacement_rows: list = [] + # columns that correspond to the replacement consequence replacement_columns = ( ds_sample.columns.get_level_values('loss') == 'replacement' ) @@ -1360,20 +1425,20 @@ def _make_replacement_exclusive( if not rows_df.empty: replacement_rows = ( - np.argwhere(np.any(rows_df.values > 0.0, axis=1)) + np.argwhere(np.any(rows_df.to_numpy() > 0.0, axis=1)) .reshape(-1) .tolist() ) - ds_sample.iloc[replacement_rows, ~replacement_columns] = 0.00 + ds_sample.iloc[replacement_rows, ~replacement_columns] = 0.00 # type: ignore if lf_sample is not None: lf_sample.iloc[replacement_rows, :] = 0.00 @property - def _loss_models(self): + def _loss_models(self) -> tuple[RepairModel_DS, RepairModel_LF]: return (self.ds_model, self.lf_model) @property - def _loss_map(self): + def _loss_map(self) -> pd.DataFrame | None: """ Returns the loss map. @@ -1385,12 +1450,12 @@ def _loss_map(self): """ # Retrieve the DataFrame from one of the included loss models. # We use a single loss map for all. - return self.ds_model._loss_map + return self.ds_model.loss_map @_loss_map.setter - def _loss_map(self, loss_map): + def _loss_map(self, loss_map: pd.DataFrame) -> None: """ - Sets the loss map. + Set the loss map. Parameters ---------- @@ -1401,10 +1466,10 @@ def _loss_map(self, loss_map): # Add the DataFrame to the included loss models. # We use a single loss map for all. for model in self._loss_models: - model._loss_map = loss_map + model.loss_map = loss_map @property - def _missing(self): + def _missing(self) -> set[tuple[str, str]]: """ Returns the missing components. @@ -1415,12 +1480,12 @@ def _missing(self): definitions. """ - return self.ds_model._missing + return self.ds_model.missing @_missing.setter - def _missing(self, missing): + def _missing(self, missing: set[tuple[str, str]]) -> None: """ - Assigns missing parameter definitions to the loss models. + Assign missing parameter definitions to the loss models. Parameters ---------- @@ -1430,24 +1495,16 @@ def _missing(self, missing): """ for model in self._loss_models: - model._missing = missing - - def _ensure_loss_parameter_availability(self) -> list: - """ - Makes sure that all components have loss parameters. - - Returns - ------- - list - List of component IDs with missing loss parameters. - - """ + model.missing = missing + def _ensure_loss_parameter_availability(self) -> None: + """Make sure that all components have loss parameters.""" # # Repair Models (currently the only type supported) # required = [] + assert self._loss_map is not None for dv in self.decision_variables: required.extend( [(component, dv) for component in self._loss_map['Repair']] @@ -1455,47 +1512,55 @@ def _ensure_loss_parameter_availability(self) -> list: missing_set = set(required) for model in (self.ds_model, self.lf_model): - missing_set = missing_set - model._get_available() + missing_set -= model.get_available() if missing_set: - self.log.warn( - f"The loss model does not provide " - f"loss information for the following component(s) " - f"in the asset model: {sorted(list(missing_set))}." + self.log.warning( + f'The loss model does not provide ' + f'loss information for the following component(s) ' + f'in the asset model: {sorted(missing_set)}.' ) self._missing = missing_set class RepairModel_Base(PelicunModel): - """ - Base class for loss models + """Base class for loss models.""" - """ - - __slots__ = ['loss_params', 'sample', 'consequence'] + __slots__ = [ + 'consequence', + 'decision_variables', + 'loss_map', + 'loss_params', + 'missing', + 'sample', + ] - def __init__(self, assessment: Assessment): + def __init__(self, assessment: AssessmentBase) -> None: """ - Initializes RepairModel_Base objects. + Initialize RepairModel_Base objects. Parameters ---------- - assessment: pelicun.Assessment + assessment: pelicun.AssessmentBase Parent assessment """ super().__init__(assessment) - self.loss_params = None - self.sample = None + self.loss_params: pd.DataFrame | None = None + self.sample: pd.DataFrame | None = None self.consequence = 'Repair' + self.decision_variables: tuple[str, ...] = () + self.loss_map: pd.DataFrame | None = None + self.missing: set = set() - def _load_model_parameters(self, data: pd.DataFrame) -> None: + def load_model_parameters(self, data: pd.DataFrame) -> None: """ - Load model parameters from a DataFrame, extending those - already available. Parameters already defined take precedence, - i.e. redefinitions of parameters are ignored. + Load model parameters from a DataFrame. + + Extends those already available. Parameters already defined + take precedence, i.e. redefinitions of parameters are ignored. Parameters ---------- @@ -1503,7 +1568,6 @@ def _load_model_parameters(self, data: pd.DataFrame) -> None: Data with loss model information. """ - data.index.names = ['Loss Driver', 'Decision Variable'] if self.loss_params is not None: @@ -1518,14 +1582,15 @@ def _load_model_parameters(self, data: pd.DataFrame) -> None: self.loss_params = data - def _drop_unused_loss_parameters(self, loss_map: pd.DataFrame) -> None: + def drop_unused_loss_parameters(self, loss_map: pd.DataFrame) -> None: """ - Removes loss parameter definitions for component IDs not - present in the loss map. + Remove loss parameter definitions. + + Applicable to component IDs not present in the loss map. Parameters ---------- - loss_map_path: str or pd.DataFrame or None + loss_map: str or pd.DataFrame or None Path to a csv file or DataFrame object that maps components IDs to their loss parameter definitions. Components in the asset model that are omitted from the @@ -1533,15 +1598,14 @@ def _drop_unused_loss_parameters(self, loss_map: pd.DataFrame) -> None: """ - if self.loss_params is None: return # if 'BldgRepair' in loss_map.columns: loss_map['Repair'] = loss_map['BldgRepair'] - loss_map.drop('BldgRepair', axis=1, inplace=True) - self.log.warn( + loss_map = loss_map.drop('BldgRepair', axis=1) + self.log.warning( '`BldgRepair` as a loss map column name is ' 'deprecated and will be dropped in ' 'future versions of pelicun. Please use `Repair` instead.' @@ -1553,8 +1617,10 @@ def _drop_unused_loss_parameters(self, loss_map: pd.DataFrame) -> None: cmp_mask = self.loss_params.index.get_level_values(0).isin(cmp_set, level=0) self.loss_params = self.loss_params.iloc[cmp_mask, :] - def _remove_incomplete_components(self) -> None: + def remove_incomplete_components(self) -> None: """ + Remove incomplete components. + Removes components that have incomplete loss model definitions from the loss model parameters. @@ -1566,48 +1632,51 @@ def _remove_incomplete_components(self) -> None: return cmp_incomplete_idx = self.loss_params.loc[ - self.loss_params[('Incomplete', '')] == 1 + self.loss_params['Incomplete', ''] == 1 ].index - self.loss_params.drop(cmp_incomplete_idx, inplace=True) + self.loss_params = self.loss_params.drop(cmp_incomplete_idx) if len(cmp_incomplete_idx) > 0: self.log.msg( - f"\n" - f"WARNING: Loss model information is incomplete for " - f"the following component(s) " - f"{cmp_incomplete_idx.to_list()}. They " - f"were removed from the analysis." - f"\n", + f'\n' + f'WARNING: Loss model information is incomplete for ' + f'the following component(s) ' + f'{cmp_incomplete_idx.to_list()}. They ' + f'were removed from the analysis.' + f'\n', prepend_timestamp=False, ) - def _get_available(self) -> set: + def get_available(self) -> set: """ - Get a set of components for which loss parameters are - available. + Get a set of components with available loss parameters. + + Returns + ------- + set + Set of components with available loss parameters. """ if self.loss_params is not None: cmp_list = self.loss_params.index.to_list() return set(cmp_list) return set() + @abstractmethod + def convert_loss_parameter_units(self) -> None: + """Convert previously loaded loss parameters to base units.""" -class RepairModel_DS(RepairModel_Base): - """ - Manages repair consequences driven by components that are modeled - with discrete Damage States (DS) - """ +class RepairModel_DS(RepairModel_Base): + """Repair consequences for components with damage states.""" - __slots__ = ['decision_variables', '_loss_map', '_missing', 'RV_reg'] + __slots__ = ['RV_reg'] def save_sample( - self, filepath: str | None = None, save_units: bool = False - ) -> None | tuple[pd.DataFrame, pd.Series]: + self, filepath: str | None = None, *, save_units: bool = False + ) -> None | pd.DataFrame | tuple[pd.DataFrame, pd.Series]: """ - Saves the loss sample to a CSV file or returns it as a - DataFrame with optional units. + Save or return the loss sample. This method handles the storage of a sample of loss estimates, which can either be saved directly to a file or returned as a @@ -1619,11 +1688,11 @@ def save_sample( Parameters ---------- - filepath : str, optional + filepath: str, optional The path to the file where the loss sample should be saved. If not provided, the sample is not saved to disk but returned. - save_units : bool, default: False + save_units: bool, default: False Indicates whether to include a row with unit information in the returned DataFrame. This parameter is ignored if a file path is provided. @@ -1638,76 +1707,121 @@ def save_sample( * Optionally, a Series containing the units for each column if `save_units` is True. - Raises - ------ - IOError - Raises an IOError if there is an issue saving the file to - the specified `filepath`. - """ - self.log.div() if filepath is not None: self.log.msg('Saving loss sample...') - cmp_units = self.loss_params[('DV', 'Unit')] - dv_units = pd.Series(index=self.sample.columns, name='Units', dtype='object') + assert self.sample is not None + assert self.loss_params is not None + cmp_units = self.loss_params['DV', 'Unit'].sort_index() + dv_units = pd.Series( + index=self.sample.columns, name='Units', dtype='object' + ).sort_index() valid_dv_types = dv_units.index.unique(level=0) valid_cmp_ids = dv_units.index.unique(level=1) for cmp_id, dv_type in cmp_units.index: if (dv_type in valid_dv_types) and (cmp_id in valid_cmp_ids): - dv_units.loc[(dv_type, cmp_id)] = cmp_units.at[(cmp_id, dv_type)] + dv_units.loc[dv_type, cmp_id] = cmp_units.loc[cmp_id, dv_type] res = file_io.save_to_csv( self.sample, - filepath, + Path(filepath) if filepath is not None else None, units=dv_units, unit_conversion_factors=self._asmnt.unit_conversion_factors, use_simpleindex=(filepath is not None), log=self._asmnt.log, ) - if filepath is not None: self.log.msg('Loss sample successfully saved.', prepend_timestamp=False) return None - units = res.loc["Units"] - res.drop("Units", inplace=True) + assert isinstance(res, pd.DataFrame) + + units = res.loc['Units'] + res = res.drop('Units') + res = res.astype(float) + assert isinstance(res, pd.DataFrame) + assert isinstance(units, pd.Series) if save_units: - return res.astype(float), units + return res, units - return res.astype(float) + return res - def load_sample(self, filepath: str | pd.DataFrame) -> None: + def load_sample(self, filepath: str | pd.DataFrame) -> dict[str, str]: """ - Load damage sample data. + Load loss sample data. + + Parameters + ---------- + filepath: str + Path to an existing sample stored in a file, or dataframe + containing the existing sample. + + Returns + ------- + dict[str, str] + Dictionary mapping each decision variable to its assigned + unit. + + Raises + ------ + ValueError + If the columns have an invalid number of levels. """ + names = ['dv', 'loss', 'dmg', 'ds', 'loc', 'dir', 'uid'] self.log.div() self.log.msg('Loading loss sample...') - self.sample = file_io.load_data( - filepath, self._asmnt.unit_conversion_factors, log=self._asmnt.log + sample, units = file_io.load_data( + filepath, + self._asmnt.unit_conversion_factors, + log=self._asmnt.log, + return_units=True, ) - self.sample.columns.names = [ - 'dv', - 'loss', - 'dmg', - 'loc', - 'dir', - 'uid', - 'block', - ] + assert isinstance(sample, pd.DataFrame) + assert isinstance(units, pd.Series) + units.index.names = names + # Obtain the DV units + # Note: we don't need to check for consistency (all rows + # having the same unit) since the units are extracted from a + # single row in the CSV, affecting all subsequent rows. + units_isolated = ( + units.reset_index()[['dv', 'Units']] + .set_index('dv') + .groupby('dv')['Units'] + ) + dv_units = units_isolated.first().to_dict() + + # check if `uid` level was provided + num_levels = len(sample.columns.names) + num_levels_without_uid = 6 + num_levels_with_uid = num_levels_without_uid + 1 + if num_levels == num_levels_without_uid: + sample.columns.names = names[:-1] + sample = base.dedupe_index(sample.T).T + elif num_levels == num_levels_with_uid: + sample.columns.names = names + else: + msg = ( + f'Invalid loss sample: Column MultiIndex ' + f'has an unexpected length: {num_levels}' + ) + raise ValueError(msg) + + self.sample = sample self.log.msg('Loss sample successfully loaded.', prepend_timestamp=False) - def _calculate(self, dmg_quantities: pd.DataFrame) -> None: + return dv_units + + def calculate(self, dmg_quantities: pd.DataFrame) -> None: # noqa: C901 """ - Calculate the damage consequences of each damage state-driven - performance group in the asset. + Calculate damage consequences. Parameters ---------- @@ -1717,12 +1831,8 @@ def _calculate(self, dmg_quantities: pd.DataFrame) -> None: and direction. You can use the prepare_dmg_quantities method in the DamageModel to get such a DF. - Raises - ------ - ValueError - When any Loss Driver is not recognized. - """ + assert self.loss_map is not None sample_size = len(dmg_quantities) @@ -1730,16 +1840,16 @@ def _calculate(self, dmg_quantities: pd.DataFrame) -> None: if set(dmg_quantities.columns.get_level_values('ds')) == {'0'}: self.sample = None self.log.msg( - "There is no damage---DV sample is set to None.", + 'There is no damage---DV sample is set to None.', prepend_timestamp=False, ) return # calculate the quantities for economies of scale - self.log.msg("\nAggregating damage quantities...", prepend_timestamp=False) + self.log.msg('\nAggregating damage quantities...', prepend_timestamp=False) - if self._asmnt.options.eco_scale["AcrossFloors"]: - if self._asmnt.options.eco_scale["AcrossDamageStates"]: + if self._asmnt.options.eco_scale['AcrossFloors']: + if self._asmnt.options.eco_scale['AcrossDamageStates']: eco_levels = [0] eco_columns = ['cmp'] @@ -1747,7 +1857,7 @@ def _calculate(self, dmg_quantities: pd.DataFrame) -> None: eco_levels = [0, 4] eco_columns = ['cmp', 'ds'] - elif self._asmnt.options.eco_scale["AcrossDamageStates"]: + elif self._asmnt.options.eco_scale['AcrossDamageStates']: eco_levels = [0, 1] eco_columns = ['cmp', 'loc'] @@ -1755,42 +1865,43 @@ def _calculate(self, dmg_quantities: pd.DataFrame) -> None: eco_levels = [0, 1, 4] eco_columns = ['cmp', 'loc', 'ds'] - eco_group = dmg_quantities.groupby(level=eco_levels, axis=1) + eco_group = dmg_quantities.groupby(level=eco_levels, axis=1) # type: ignore eco_qnt = eco_group.sum().mask(eco_group.count() == 0, np.nan) assert eco_qnt.columns.names == eco_columns self.log.msg( - "Successfully aggregated damage quantities.", prepend_timestamp=False + 'Successfully aggregated damage quantities.', prepend_timestamp=False ) # apply the median functions, if needed, to get median consequences for # each realization self.log.msg( - "\nCalculating the median repair consequences...", + '\nCalculating the median repair consequences...', prepend_timestamp=False, ) medians = self._calc_median_consequence(eco_qnt) self.log.msg( - "Successfully determined median repair consequences.", + 'Successfully determined median repair consequences.', prepend_timestamp=False, ) # combine the median consequences with the samples of deviation from the # median to get the consequence realizations. self.log.msg( - "\nConsidering deviations from the median values to obtain " - "random DV sample..." + '\nConsidering deviations from the median values to obtain ' + 'random DV sample...' ) self.log.msg( - "Preparing random variables for repair consequences...", + 'Preparing random variables for repair consequences...', prepend_timestamp=False, ) - self.RV_reg = self._create_DV_RVs(dmg_quantities.columns) + self.RV_reg = self._create_DV_RVs(dmg_quantities.columns) # type: ignore if self.RV_reg is not None: + assert self._asmnt.options.sampling_method is not None self.RV_reg.generate_sample( sample_size=sample_size, method=self._asmnt.options.sampling_method ) @@ -1799,45 +1910,46 @@ def _calculate(self, dmg_quantities: pd.DataFrame) -> None: pd.DataFrame(self.RV_reg.RV_sample), axis=1 ) std_sample.columns.names = ['dv', 'cmp', 'ds', 'loc', 'dir', 'uid'] - std_sample.sort_index(axis=1, inplace=True) + std_sample = std_sample.sort_index(axis=1) else: std_sample = None self.log.msg( - f"\nSuccessfully generated {sample_size} realizations of " - "deviation from the median consequences.", + f'\nSuccessfully generated {sample_size} realizations of ' + 'deviation from the median consequences.', prepend_timestamp=False, ) res_list = [] - key_list = [] + key_list: list[tuple[Any, ...]] = [] - dmg_quantities.columns = dmg_quantities.columns.reorder_levels( + dmg_quantities.columns = dmg_quantities.columns.reorder_levels( # type: ignore ['cmp', 'ds', 'loc', 'dir', 'uid'] ) - dmg_quantities.sort_index(axis=1, inplace=True) + dmg_quantities = dmg_quantities.sort_index(axis=1) if std_sample is not None: std_dvs = std_sample.columns.unique(level=0) else: std_dvs = [] - for decision_variable in self.decision_variables: + for decision_variable in self.decision_variables: # noqa: PLR1702 if decision_variable in std_dvs: + assert isinstance(std_sample, pd.DataFrame) prob_cmp_list = std_sample[decision_variable].columns.unique(level=0) else: prob_cmp_list = [] - cmp_list = [] + cmp_list: list[tuple[Any, ...]] = [] if decision_variable not in medians: continue for component in medians[decision_variable].columns.unique(level=0): # check if there is damage in the component - consequence = self._loss_map.at[component, 'Repair'] + consequence = self.loss_map.loc[component, 'Repair'] - if not (component in dmg_quantities.columns.get_level_values('cmp')): + if component not in dmg_quantities.columns.get_level_values('cmp'): continue ds_list = [] @@ -1850,24 +1962,32 @@ def _calculate(self, dmg_quantities: pd.DataFrame) -> None: loc_list = [] for loc_id, loc in enumerate( - dmg_quantities.loc[:, (component, ds)].columns.unique( - level=0 - ) + dmg_quantities.loc[ + :, (component, ds) # type: ignore + ].columns.unique(level=0) ): if ( - self._asmnt.options.eco_scale["AcrossFloors"] is True + self._asmnt.options.eco_scale['AcrossFloors'] is True ) and (loc_id > 0): break - if self._asmnt.options.eco_scale["AcrossFloors"] is True: + if self._asmnt.options.eco_scale['AcrossFloors'] is True: median_i = medians[decision_variable].loc[ :, (component, ds) ] - dmg_i = dmg_quantities.loc[:, (component, ds)] + dmg_i = dmg_quantities.loc[ + :, (component, ds) # type: ignore + ] if component in prob_cmp_list: + assert std_sample is not None std_i = std_sample.loc[ - :, (decision_variable, component, ds) + :, + ( + decision_variable, + component, + ds, + ), # type: ignore ] else: std_i = None @@ -1876,11 +1996,20 @@ def _calculate(self, dmg_quantities: pd.DataFrame) -> None: median_i = medians[decision_variable].loc[ :, (component, ds, loc) ] - dmg_i = dmg_quantities.loc[:, (component, ds, loc)] + dmg_i = dmg_quantities.loc[ + :, (component, ds, loc) # type: ignore + ] if component in prob_cmp_list: + assert std_sample is not None std_i = std_sample.loc[ - :, (decision_variable, component, ds, loc) + :, + ( + decision_variable, + component, + ds, + loc, + ), # type: ignore ] else: std_i = None @@ -1892,21 +2021,21 @@ def _calculate(self, dmg_quantities: pd.DataFrame) -> None: loc_list.append(loc) - if self._asmnt.options.eco_scale["AcrossFloors"] is True: + if self._asmnt.options.eco_scale['AcrossFloors'] is True: ds_list += [ ds, ] else: ds_list += [(ds, loc) for loc in loc_list] - if self._asmnt.options.eco_scale["AcrossFloors"] is True: + if self._asmnt.options.eco_scale['AcrossFloors'] is True: cmp_list += [(consequence, component, ds) for ds in ds_list] else: cmp_list += [ (consequence, component, ds, loc) for ds, loc in ds_list ] - if self._asmnt.options.eco_scale["AcrossFloors"] is True: + if self._asmnt.options.eco_scale['AcrossFloors'] is True: key_list += [ (decision_variable, loss_cmp_i, dmg_cmp_i, ds) for loss_cmp_i, dmg_cmp_i, ds in cmp_list @@ -1918,32 +2047,33 @@ def _calculate(self, dmg_quantities: pd.DataFrame) -> None: ] lvl_names = ['dv', 'loss', 'dmg', 'ds', 'loc', 'dir', 'uid'] - DV_sample = pd.concat(res_list, axis=1, keys=key_list, names=lvl_names) + dv_sample = pd.concat(res_list, axis=1, keys=key_list, names=lvl_names) - DV_sample = DV_sample.fillna(0).convert_dtypes() + dv_sample = dv_sample.fillna(0).convert_dtypes() - self.log.msg("Successfully obtained DV sample.", prepend_timestamp=False) - self.sample = DV_sample - - def _convert_loss_parameter_units(self) -> None: - """ - Converts previously loaded loss parameters to base units. + self.log.msg('Successfully obtained DV sample.', prepend_timestamp=False) + self.sample = dv_sample - """ + def convert_loss_parameter_units(self) -> None: + """Convert previously loaded loss parameters to base units.""" if self.loss_params is None: return - units = self.loss_params[('DV', 'Unit')] - arg_units = self.loss_params[('Quantity', 'Unit')] + units = self.loss_params['DV', 'Unit'] + arg_units = self.loss_params['Quantity', 'Unit'] for column in self.loss_params.columns.unique(level=0): if not column.startswith('DS'): continue + params = self.loss_params.loc[:, column].copy() + assert isinstance(params, pd.DataFrame) self.loss_params.loc[:, column] = self._convert_marginal_params( - self.loss_params.loc[:, column].copy(), units, arg_units - ).values + params, units, arg_units + ).to_numpy() - def _drop_unused_damage_states(self) -> None: + def drop_unused_damage_states(self) -> None: """ - Removes columns from the loss model parameters corresponding + Remove unused columns. + + Remove columns from the loss model parameters corresponding to unused damage states. """ @@ -1954,21 +2084,30 @@ def _drop_unused_damage_states(self) -> None: ds_to_drop = [] for damage_state in ds_list: if ( - np.all(pd.isna(self.loss_params.loc[:, idx[damage_state, :]].values)) + np.all( + pd.isna( + self.loss_params.loc[ + :, # type: ignore + idx[damage_state, :], + ].values + ) + ) # Note: When this evaluates to True, when you add `is # True` on the right it suddenly evaluates to # False. We need to figure out why this is happening, # but the way it's written now does what we want in # each case. ): - ds_to_drop.append(damage_state) + ds_to_drop.append(damage_state) # noqa: PERF401 - self.loss_params.drop(columns=ds_to_drop, level=0, inplace=True) + self.loss_params = self.loss_params.drop(columns=ds_to_drop, level=0) - def _create_DV_RVs( + def _create_DV_RVs( # noqa: N802, C901 self, cases: pd.MultiIndex ) -> uq.RandomVariableRegistry | None: """ + Prepare the random variables. + Prepare the random variables associated with decision variables, such as repair cost and time. @@ -1986,14 +2125,7 @@ def _create_DV_RVs( random variables are generated (due to missing parameters or conditions), returns None. - Raises - ------ - ValueError - If an unrecognized loss driver type is encountered, - indicating a configuration or data input error. - """ - # Convert the MultiIndex to a DataFrame case_df = pd.DataFrame(index=cases).reset_index() # maps `cmp` to array of damage states @@ -2001,18 +2133,26 @@ def _create_DV_RVs( # maps `cmp`-`ds` to tuple of `loc`-`dir`-`uid` tuples loc_dir_uids = ( case_df.groupby(['cmp', 'ds']) - .apply(lambda x: tuple(zip(x['loc'], x['dir'], x['uid']))) + .apply( + lambda x: tuple( # type: ignore + zip( + x['loc'], + x['dir'], + x['uid'], + ) + ) + ) .to_dict() ) damaged_components = set(cases.get_level_values('cmp')) - RV_reg = uq.RandomVariableRegistry(self._asmnt.options.rng) + rv_reg = uq.RandomVariableRegistry(self._asmnt.options.rng) rv_count = 0 # for each component in the loss map - for component, consequence in self._loss_map['Repair'].items(): - + assert isinstance(self.loss_map, pd.DataFrame) + for component, consequence in self.loss_map['Repair'].items(): # if that component does not have realized damage states, # skip it (e.g., this can happen when there is only # `collapse`). @@ -2021,16 +2161,16 @@ def _create_DV_RVs( # for each DV for decision_variable in self.decision_variables: - # If loss parameters are missing for that consequence, # don't estimate losses for it. A warning has already # been issued for what is missing. - if (consequence, decision_variable) in self._missing: + if (consequence, decision_variable) in self.missing: continue # If loss parameters are missing for that consequence, # for this particular loss model, they will exist in # the other(s). + assert self.loss_params is not None if (consequence, decision_variable) not in self.loss_params.index: continue @@ -2042,16 +2182,16 @@ def _create_DV_RVs( ) for ds in damage_states[component]: - if ds == '0': continue ds_family = parameters.get((f'DS{ds}', 'Family')) ds_theta = [ - parameters.get((f'DS{ds}', f'Theta_{t_i}'), np.nan) + theta for t_i in range(3) + if (theta := parameters.get((f'DS{ds}', f'Theta_{t_i}'))) + is not None ] - # If there is no RV family we don't need an RV if ds_family is None: continue @@ -2062,18 +2202,18 @@ def _create_DV_RVs( if isinstance(ds_theta[0], str) and '|' in ds_theta[0]: ds_theta[0] = 1.0 - loc_dir_uid = loc_dir_uids[(component, ds)] + loc_dir_uid = loc_dir_uids[component, ds] for loc, direction, uid in loc_dir_uid: # assign RVs - RV_reg.add_RV( - uq.rv_class_map(ds_family)( + rv_reg.add_RV( + uq.rv_class_map(ds_family)( # type: ignore name=( f'{decision_variable}-{component}-' f'{ds}-{loc}-{direction}-{uid}' ), - theta=ds_theta, - truncation_limits=[0.0, np.nan], + theta=np.array(ds_theta), + truncation_limits=np.array([0.0, np.nan]), ) ) rv_count += 1 @@ -2081,7 +2221,7 @@ def _create_DV_RVs( # assign Time-Cost correlation whenever applicable rho = self._asmnt.options.rho_cost_time if rho != 0.0: - for rv_tag in RV_reg.RV: + for rv_tag in rv_reg.RV: if not rv_tag.startswith('Cost'): continue component = rv_tag.split('-')[1] @@ -2090,26 +2230,28 @@ def _create_DV_RVs( direction = rv_tag.split('-')[4] uid = rv_tag.split('-')[5] time_rv_tag = rv_tag.replace('Cost', 'Time') - if time_rv_tag in RV_reg.RV: - RV_reg.add_RV_set( + if time_rv_tag in rv_reg.RV: + rv_reg.add_RV_set( uq.RandomVariableSet( f'DV-{component}-{ds}-{loc}-{direction}-{uid}_set', - list(RV_reg.RVs([rv_tag, time_rv_tag]).values()), + list(rv_reg.RVs([rv_tag, time_rv_tag]).values()), np.array([[1.0, rho], [rho, 1.0]]), ) ) self.log.msg( - f"\n{rv_count} random variables created.", prepend_timestamp=False + f'\n{rv_count} random variables created.', prepend_timestamp=False ) if rv_count > 0: - return RV_reg + return rv_reg return None - def _calc_median_consequence(self, eco_qnt: pd.DataFrame) -> dict: + def _calc_median_consequence(self, eco_qnt: pd.DataFrame) -> dict: # noqa: C901 """ - Calculates the median repair consequences for each loss + Calculate median reiapr consequences. + + Calculate the median repair consequences for each loss component based on its quantity realizations and the associated loss parameters. @@ -2143,17 +2285,17 @@ def _calc_median_consequence(self, eco_qnt: pd.DataFrame) -> dict: If any loss driver types or distribution types are not recognized, or if the parameters are incomplete or unsupported. - """ + """ medians = {} for decision_variable in self.decision_variables: cmp_list = [] median_list = [] - for loss_cmp_id, loss_cmp_name in self._loss_map['Repair'].items(): - - if (loss_cmp_name, decision_variable) in self._missing: + assert self.loss_map is not None + for loss_cmp_id, loss_cmp_name in self.loss_map['Repair'].items(): + if (loss_cmp_name, decision_variable) in self.missing: continue if loss_cmp_id not in eco_qnt.columns.get_level_values(0).unique(): @@ -2162,6 +2304,7 @@ def _calc_median_consequence(self, eco_qnt: pd.DataFrame) -> dict: ds_list = [] sub_medians = [] + assert self.loss_params is not None for ds in self.loss_params.columns.get_level_values(0).unique(): if not ds.startswith('DS'): continue @@ -2171,31 +2314,32 @@ def _calc_median_consequence(self, eco_qnt: pd.DataFrame) -> dict: if ds_id == '0': continue - loss_params_DS = self.loss_params.loc[ + loss_params_ds = self.loss_params.loc[ (loss_cmp_name, decision_variable), ds ] # check if theta_0 is defined - theta_0 = loss_params_DS.get('Theta_0', np.nan) + theta_0 = loss_params_ds.get('Theta_0', np.nan) if pd.isna(theta_0): continue # check if the distribution type is supported - family = loss_params_DS.get('Family', np.nan) + family = loss_params_ds.get('Family', np.nan) if (not pd.isna(family)) and ( - family not in ['normal', 'lognormal', 'deterministic'] + family not in {'normal', 'lognormal', 'deterministic'} ): - raise ValueError( - f"Loss Distribution of type {family} " f"not supported." + msg = ( + f'Loss Distribution of type {family} ' f'not supported.' ) + raise ValueError(msg) # If theta_0 is a scalar try: theta_0 = float(theta_0) - if pd.isna(loss_params_DS.get('Family', np.nan)): + if pd.isna(loss_params_ds.get('Family', np.nan)): # if theta_0 is constant, then use it directly f_median = _prep_constant_median_DV(theta_0) @@ -2218,17 +2362,24 @@ def _calc_median_consequence(self, eco_qnt: pd.DataFrame) -> dict: # get the corresponding aggregate damage quantities # to consider economies of scale if 'ds' in eco_qnt.columns.names: - avail_ds = eco_qnt.loc[:, loss_cmp_id].columns.unique( - level=0 - ) + avail_ds = eco_qnt.loc[ + :, # type: ignore + loss_cmp_id, + ].columns.unique(level=0) if ds_id not in avail_ds: continue - eco_qnt_i = eco_qnt.loc[:, (loss_cmp_id, ds_id)].copy() + eco_qnt_i = eco_qnt.loc[ + :, # type: ignore + (loss_cmp_id, ds_id), + ].copy() else: - eco_qnt_i = eco_qnt.loc[:, loss_cmp_id].copy() + eco_qnt_i = eco_qnt.loc[ + :, # type: ignore + loss_cmp_id, + ].copy() if isinstance(eco_qnt_i, pd.Series): eco_qnt_i = eco_qnt_i.to_frame() @@ -2236,7 +2387,7 @@ def _calc_median_consequence(self, eco_qnt: pd.DataFrame) -> dict: eco_qnt_i.columns.name = 'del' # generate the median values for each realization - eco_qnt_i.loc[:, :] = f_median(eco_qnt_i.values) + eco_qnt_i.loc[:, :] = f_median(eco_qnt_i.values) # type: ignore sub_medians.append(eco_qnt_i) ds_list.append(ds_id) @@ -2267,15 +2418,11 @@ def _calc_median_consequence(self, eco_qnt: pd.DataFrame) -> dict: class RepairModel_LF(RepairModel_Base): - """ - Manages repair consequences driven by components that are modeled - with Loss Functions (LF) - - """ + """Repair consequences for components with loss functions.""" - __slots__ = ['decision_variables', '_loss_map', '_missing'] + __slots__ = [] - def _calculate( + def calculate( self, demand_sample: pd.DataFrame, cmp_sample: dict, @@ -2284,29 +2431,36 @@ def _calculate( nondirectional_multipliers: dict, ) -> None: """ - Calculate the repair consequences of each loss function-driven - component block in the asset. + Calculate repair consequences. Parameters ---------- demand_sample: pd.DataFrame The sample of the demand model to be used for the inputs of the loss functions. + cmp_sample: dict + Dict mapping each `cmp`-`loc`-`dir`-`uid` to the component + quantity realizations in the asset model in the form of + pd.Series objects. + cmp_marginal_params: pd.DataFrame + Dataframe containing component marginal distribution + parameters. + demand_offset: dict + Dictionary specifying the demand offset. + nondirectional_multipliers: dict + Dictionary specifying the non directional multipliers used + to combine the directional demands. - Raises - ------ - ValueError - When any Loss Driver is not recognized. """ - if self.loss_params is None: - return None + return - loss_map = self._loss_map['Repair'].to_dict() + assert self.loss_map is not None + loss_map = self.loss_map['Repair'].to_dict() sample_size = len(demand_sample) - # TODO: this can be taken out and simply passed as blocks in + # TODO(JVM): this can be taken out and simply passed as blocks in # the arguments, and cast to a dict in here. Index can be # obtained from there. index = [ @@ -2323,20 +2477,20 @@ def _calculate( performance_group_dict = {} for (component, location, direction, uid), num_blocks in blocks.items(): for decision_variable in self.decision_variables: - if (component, decision_variable) in self._missing: + if (component, decision_variable) in self.missing: continue performance_group_dict[ - ((component, decision_variable), location, direction, uid) + (component, decision_variable), location, direction, uid ] = num_blocks if not performance_group_dict: self.log.msg( - "No loss function-driven components---LF sample is set to None.", + 'No loss function-driven components---LF sample is set to None.', prepend_timestamp=False, ) - return None + return - performance_group = pd.DataFrame( + performance_group = pd.DataFrame( # type: ignore performance_group_dict.values(), index=performance_group_dict.keys(), columns=['Blocks'], @@ -2367,7 +2521,7 @@ def _calculate( ) self.log.msg( - "\nCalculating the median repair consequences...", + '\nCalculating the median repair consequences...', prepend_timestamp=False, ) @@ -2376,28 +2530,29 @@ def _calculate( ) self.log.msg( - "Successfully determined median repair consequences.", + 'Successfully determined median repair consequences.', prepend_timestamp=False, ) self.log.msg( - "\nConsidering deviations from the median values to obtain " - "random DV sample..." + '\nConsidering deviations from the median values to obtain ' + 'random DV sample...' ) self.log.msg( - "Preparing random variables for repair cost and time...", + 'Preparing random variables for repair cost and time...', prepend_timestamp=False, ) - RV_reg = self._create_DV_RVs(medians.columns) - if RV_reg is not None: - RV_reg.generate_sample( + rv_reg = self._create_DV_RVs(medians.columns) # type: ignore + if rv_reg is not None: + assert self._asmnt.options.sampling_method is not None + rv_reg.generate_sample( sample_size=sample_size, method=self._asmnt.options.sampling_method ) std_sample = base.convert_to_MultiIndex( - pd.DataFrame(RV_reg.RV_sample), axis=1 + pd.DataFrame(rv_reg.RV_sample), axis=1 ) std_sample.columns.names = [ 'dv', @@ -2408,42 +2563,44 @@ def _calculate( 'uid', 'block', ] - std_sample.sort_index(axis=1, inplace=True) + std_sample = std_sample.sort_index(axis=1) sample = (medians * std_sample).combine_first(medians) else: sample = medians self.log.msg( - f"\nSuccessfully generated {sample_size} realizations of " - "deviation from the median consequences.", + f'\nSuccessfully generated {sample_size} realizations of ' + 'deviation from the median consequences.', prepend_timestamp=False, ) # sum up the block losses - sample = sample.groupby( + sample = sample.groupby( # type: ignore by=['dv', 'loss', 'dmg', 'loc', 'dir', 'uid'], axis=1 ).sum() - self.log.msg("Successfully obtained DV sample.", prepend_timestamp=False) + self.log.msg('Successfully obtained DV sample.', prepend_timestamp=False) self.sample = sample - return None - - def _convert_loss_parameter_units(self) -> None: - """ - Converts previously loaded loss parameters to base units. + return - """ + def convert_loss_parameter_units(self) -> None: + """Convert previously loaded loss parameters to base units.""" if self.loss_params is None: - return None - units = self.loss_params[('DV', 'Unit')] - arg_units = self.loss_params[('Demand', 'Unit')] + return + units = self.loss_params['DV', 'Unit'] + arg_units = self.loss_params['Demand', 'Unit'] column = 'LossFunction' + params = self.loss_params[column].copy() + assert isinstance(params, pd.DataFrame) self.loss_params.loc[:, column] = self._convert_marginal_params( - self.loss_params[column].copy(), units, arg_units, divide_units=False - ).values - return None + params, + units, + arg_units, + divide_units=False, + ).to_numpy() + return def _calc_median_consequence( self, @@ -2454,6 +2611,8 @@ def _calc_median_consequence( cmp_sample: dict, ) -> pd.DataFrame: """ + Calculate median repair consequences. + Calculates the median repair consequences for each loss function-driven component based on its quantity realizations and the associated loss parameters. @@ -2487,12 +2646,22 @@ def _calc_median_consequence( Dataframe with medial loss for loss-function driven components. - """ + Raises + ------ + ValueError + If loss function interpolation fails. + """ medians_dict = {} # for each component in the asset model - for ( + component: str + decision_variable: str + location: str + direction: str + uid: str + blocks: int + for ( # type: ignore (component, decision_variable), location, direction, @@ -2500,50 +2669,51 @@ def _calc_median_consequence( ), blocks in performance_group['Blocks'].items(): consequence = loss_map[component] edp = required_edps[ - ((consequence, decision_variable), location, direction, uid) + (consequence, decision_variable), location, direction, uid ] edp_values = demand_dict[edp] - loss_function_str = self.loss_params.at[ + assert self.loss_params is not None + loss_function_str = self.loss_params.loc[ (component, decision_variable), ('LossFunction', 'Theta_0') ] + assert isinstance(loss_function_str, str) try: median_loss = base.stringterpolation(loss_function_str)(edp_values) except ValueError as exc: - raise ValueError( + msg = ( f'Loss function interpolation for consequence ' f'`{consequence}-{decision_variable}` has failed. ' f'Ensure a sufficient interpolation domain ' f'for the X values (those after the `|` symbol) ' f'and verify the X-value and Y-value lengths match.' - ) from exc + ) + raise ValueError(msg) from exc for block in range(blocks): medians_dict[ - ( - decision_variable, - consequence, - component, - location, - direction, - uid, - str(block), - ) + decision_variable, + consequence, + component, + location, + direction, + uid, + str(block), ] = ( median_loss - * cmp_sample[component, location, direction, uid].values + * cmp_sample[component, location, direction, uid].to_numpy() / float(blocks) ) medians = pd.DataFrame(medians_dict) medians.columns.names = ['dv', 'loss', 'dmg', 'loc', 'dir', 'uid', 'block'] - medians.sort_index(axis=1, inplace=True) + return medians.sort_index(axis=1) - return medians - - def _create_DV_RVs( + def _create_DV_RVs( # noqa: N802 self, cases: pd.MultiIndex ) -> uq.RandomVariableRegistry | None: """ - Prepare the random variables associated with decision + Prepare the decision variable random variables. + + Prepares the random variables associated with decision variables, such as repair cost and time. Parameters @@ -2562,8 +2732,7 @@ def _create_DV_RVs( or conditions), returns None. """ - - RV_reg = uq.RandomVariableRegistry(self._asmnt.options.rng) + rv_reg = uq.RandomVariableRegistry(self._asmnt.options.rng) rv_count = 0 @@ -2577,19 +2746,20 @@ def _create_DV_RVs( uid, block, ) in cases: - # load the corresponding parameters + assert self.loss_params is not None parameters = self.loss_params.loc[(consequence, decision_variable), :] if ('LossFunction', 'Family') not in parameters: # Everything is deterministic, no need to create RVs. continue - family = parameters.at[('LossFunction', 'Family')] + family = parameters.loc['LossFunction', 'Family'] theta = [ - parameters.get(('LossFunction', f'Theta_{t_i}'), np.nan) + theta_value for t_i in range(3) + if (theta_value := parameters.get(('LossFunction', f'Theta_{t_i}'))) + is not None ] - # If there is no RV family we don't need an RV if pd.isna(family): continue @@ -2597,17 +2767,17 @@ def _create_DV_RVs( # Since the first parameter is controlled by a function, # we use 1.0 in its place and will scale the results in a # later step. - theta[0] = 1.0 + theta[0] = 1.0 # type: ignore # assign RVs - RV_reg.add_RV( - uq.rv_class_map(family)( + rv_reg.add_RV( + uq.rv_class_map(family)( # type: ignore name=( f'{decision_variable}-{consequence}-' f'{component}-{location}-{direction}-{uid}-{block}' ), - theta=theta, - truncation_limits=[0.0, np.nan], + theta=np.array(theta), + truncation_limits=np.array([0.0, np.nan]), ) ) rv_count += 1 @@ -2615,7 +2785,7 @@ def _create_DV_RVs( # assign Time-Cost correlation whenever applicable rho = self._asmnt.options.rho_cost_time if rho != 0.0: - for rv_tag in RV_reg.RV: + for rv_tag in rv_reg.RV: if not rv_tag.startswith('Cost'): continue split = rv_tag.split('-') @@ -2626,30 +2796,30 @@ def _create_DV_RVs( uid = split[5] block = split[6] time_rv_tag = rv_tag.replace('Cost', 'Time') - if time_rv_tag in RV_reg.RV: - RV_reg.add_RV_set( + if time_rv_tag in rv_reg.RV: + rv_reg.add_RV_set( uq.RandomVariableSet( ( f'DV-{consequence}-{component}-' f'{location}-{direction}-{uid}-{block}_set' ), - list(RV_reg.RVs([rv_tag, time_rv_tag]).values()), + list(rv_reg.RVs([rv_tag, time_rv_tag]).values()), np.array([[1.0, rho], [rho, 1.0]]), ) ) self.log.msg( - f"\n{rv_count} random variables created.", prepend_timestamp=False + f'\n{rv_count} random variables created.', prepend_timestamp=False ) if rv_count > 0: - return RV_reg + return rv_reg return None -def _prep_constant_median_DV(median: float) -> callable: +def _prep_constant_median_DV(median: float) -> Callable: # noqa: N802 """ - Returns a constant median Decision Variable (DV) function. + Return a constant median Decision Variable (DV) function. Parameters ---------- @@ -2664,20 +2834,17 @@ def _prep_constant_median_DV(median: float) -> callable: """ - def f(*args): - # pylint: disable=unused-argument - # pylint: disable=missing-return-doc - # pylint: disable=missing-return-type-doc + def f(*args): # noqa: ANN002, ANN202, ARG001 return median return f -def _prep_bounded_multilinear_median_DV( +def _prep_bounded_multilinear_median_DV( # noqa: N802 medians: np.ndarray, quantities: np.ndarray -) -> callable: +) -> Callable: """ - Returns a bounded multilinear median Decision Variable (DV) function. + Return a bounded multilinear median Decision Variable (DV) function. The median DV equals the min and max values when the quantity is outside of the prescribed quantity bounds. When the quantity is within the @@ -2698,39 +2865,57 @@ def _prep_bounded_multilinear_median_DV( callable A function that returns the median DV given the quantity of damaged components. + """ - def f(quantity): - # pylint: disable=missing-return-doc - # pylint: disable=missing-return-type-doc + def f(quantity): # noqa: ANN001, ANN202 if quantity is None: - raise ValueError( + msg = ( 'A bounded linear median Decision Variable function called ' 'without specifying the quantity of damaged components' ) + raise ValueError(msg) q_array = np.asarray(quantity, dtype=np.float64) # calculate the median consequence given the quantity of damaged # components - output = np.interp(q_array, quantities, medians) - - return output + return np.interp(q_array, quantities, medians) return f def _is_for_lf_model(data: pd.DataFrame) -> bool: """ - Determines if the specified loss model parameters are for - components modeled with Loss Functions (LF). + Determine if the data are for the lf_model. + + Parameters + ---------- + data: pd.DataFrame + Data to be checked. + + Returns + ------- + bool + Whether the data are for the lf_model. + """ return 'LossFunction' in data.columns.get_level_values(0) def _is_for_ds_model(data: pd.DataFrame) -> bool: """ - Determines if the specified loss model parameters are for - components modeled with discrete Damage States (DS). + Determine if the data are for the ds_model. + + Parameters + ---------- + data: pd.DataFrame + Data to be checked. + + Returns + ------- + bool + Whether the data are for the ds_model. + """ return 'DS1' in data.columns.get_level_values(0) diff --git a/pelicun/model/pelicun_model.py b/pelicun/model/pelicun_model.py index 6fbf3799f..767a8e1ff 100644 --- a/pelicun/model/pelicun_model.py +++ b/pelicun/model/pelicun_model.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -38,56 +37,58 @@ # Adam Zsarnóczay # John Vouvakis Manousakis -""" -This file defines the PelicunModel object and its methods. -.. rubric:: Contents +"""PelicunModel object and associated methods.""" -.. autosummary:: - - PelicunModel +from __future__ import annotations -""" +from typing import TYPE_CHECKING, Any -from __future__ import annotations -from typing import TYPE_CHECKING import numpy as np import pandas as pd -from pelicun import base -from pelicun import uq + +from pelicun import base, uq if TYPE_CHECKING: - from pelicun.assessment import Assessment + from pelicun.assessment import AssessmentBase + from pelicun.base import Logger idx = base.idx class PelicunModel: - """ - Generic model class to manage methods shared between all models in Pelicun. - - """ + """Generic model class to manage methods shared between all models in Pelicun.""" __slots__ = ['_asmnt', 'log'] - def __init__(self, assessment: Assessment): - # link the PelicunModel object to its Assessment object - self._asmnt: Assessment = assessment + def __init__(self, assessment: AssessmentBase) -> None: + """ + Instantiate PelicunModel objects. + + Parameters + ---------- + assessment: Assessment + Parent assessment object. + + """ + # link the PelicunModel object to its AssessmentBase object + self._asmnt: AssessmentBase = assessment # link logging methods as attributes enabling more # concise syntax - self.log = self._asmnt.log + self.log: Logger = self._asmnt.log - def _convert_marginal_params( + def _convert_marginal_params( # noqa: C901 self, marginal_params: pd.DataFrame, units: pd.Series, arg_units: pd.Series | None = None, + *, divide_units: bool = True, inverse_conversion: bool = False, ) -> pd.DataFrame: """ - Converts the parameters of marginal distributions in a model to SI units. + Convert the parameters of marginal distributions in a model to SI units. Parameters ---------- @@ -148,10 +149,10 @@ def _convert_marginal_params( marginal_params[col_name] = np.nan # get a list of unique units - unique_units = units.unique() + unique_units = units.dropna().unique() # for each unit - for unit_name in unique_units: + for unit_name in unique_units: # noqa: PLR1702 # get the scale factor for converting from the source unit unit_factor = self._asmnt.calc_unit_scale_factor(unit_name) @@ -161,7 +162,7 @@ def _convert_marginal_params( # for each variable for row_id in unit_ids: # pull the parameters of the marginal distribution - family = marginal_params.at[row_id, 'Family'] + family = marginal_params.loc[row_id, 'Family'] if family == 'empirical': continue @@ -169,10 +170,10 @@ def _convert_marginal_params( # load the theta values theta = marginal_params.loc[ row_id, ['Theta_0', 'Theta_1', 'Theta_2'] - ].values + ].to_numpy() # for each theta - args = [] + args: list[Any] = [] for t_i, theta_i in enumerate(theta): # if theta_i evaluates to NaN, it is considered undefined if pd.isna(theta_i): @@ -206,9 +207,10 @@ def _convert_marginal_params( arg_unit_factor = 1.0 # check if there is a need to scale due to argument units - if not (arg_units is None): + if arg_units is not None: # get the argument unit for the given marginal arg_unit = arg_units.get(row_id) + assert isinstance(arg_unit, str) if arg_unit != '1 EA': # get the scale factor @@ -228,8 +230,11 @@ def _convert_marginal_params( conversion_factor = unit_factor if inverse_conversion: conversion_factor = 1.00 / conversion_factor - theta, tr_limits = uq.scale_distribution( - conversion_factor, family, theta, tr_limits + theta, tr_limits = uq.scale_distribution( # type: ignore + conversion_factor, + family, + theta, + tr_limits, # type: ignore ) # convert multilinear function parameters back into strings @@ -238,7 +243,7 @@ def _convert_marginal_params( theta[a_i] = '|'.join( [ ','.join([f'{val:g}' for val in vals]) - for vals in (theta[a_i], args[a_i]) + for vals in (theta[a_i], arg) ] ) @@ -252,14 +257,14 @@ def _convert_marginal_params( ) # remove the added columns - marginal_params = marginal_params[original_cols] - - return marginal_params + return marginal_params[original_cols] def _get_locations(self, loc_str: str) -> np.ndarray: """ - Parses a location string to determine specific sections of - an asset to be processed. + Parse a location string. + + Parses a location string to determine specific sections of an + asset to be processed. This function interprets various string formats to output a list of strings representing sections or parts of the @@ -269,7 +274,7 @@ def _get_locations(self, loc_str: str) -> np.ndarray: Parameters ---------- - loc_str : str + loc_str: str A string that describes the location or range of sections in the asset. It can be a single number, a range, a comma-separated list, 'all', 'top', or @@ -311,38 +316,45 @@ def _get_locations(self, loc_str: str) -> np.ndarray: >>> _get_locations('roof') array(['11']) + """ try: - res = str(int(loc_str)) + res = str(int(float(loc_str))) return np.array([res]) except ValueError as exc: stories = self._asmnt.stories - if "--" in loc_str: + if '--' in loc_str: s_low, s_high = loc_str.split('--') - s_low = self._get_locations(s_low) - s_high = self._get_locations(s_high) - return np.arange(int(s_low[0]), int(s_high[0]) + 1).astype(str) + s_low = self._get_locations(s_low)[0] + s_high = self._get_locations(s_high)[0] + return np.arange(int(s_low), int(s_high) + 1).astype(str) - if "," in loc_str: + if ',' in loc_str: return np.array(loc_str.split(','), dtype=int).astype(str) - if loc_str == "all": + if loc_str == 'all': + assert stories is not None return np.arange(1, stories + 1).astype(str) - if loc_str == "top": + if loc_str == 'top': + assert stories is not None return np.array([stories]).astype(str) - if loc_str == "roof": + if loc_str == 'roof': + assert stories is not None return np.array([stories + 1]).astype(str) - raise ValueError(f"Cannot parse location string: " f"{loc_str}") from exc + msg = f'Cannot parse location string: ' f'{loc_str}' + raise ValueError(msg) from exc def _get_directions(self, dir_str: str | None) -> np.ndarray: """ - Parses a direction string to determine specific - orientations or directions applicable within an asset. + Parse a direction string. + + Parses a direction string to determine specific orientations + or directions applicable within an asset. This function processes direction descriptions to output an array of strings, each representing a specific @@ -352,7 +364,7 @@ def _get_directions(self, dir_str: str | None) -> np.ndarray: Parameters ---------- - dir_str : str or None + dir_str: str or None A string that describes the direction or range of directions in the asset. It can be a single number, a range, a comma-separated list, or it can be null, @@ -389,25 +401,57 @@ def _get_directions(self, dir_str: str | None) -> np.ndarray: >>> get_directions('1,2,5') array(['1', '2', '5']) + """ - if pd.isnull(dir_str): + if pd.isna(dir_str): return np.ones(1).astype(str) try: - res = str(int(dir_str)) + res = str(int(float(dir_str))) # type: ignore return np.array([res]) except ValueError as exc: - if "," in dir_str: - return np.array(dir_str.split(','), dtype=int).astype(str) - - if "--" in dir_str: - d_low, d_high = dir_str.split('--') - d_low = self._get_directions(d_low) - d_high = self._get_directions(d_high) - return np.arange(int(d_low[0]), int(d_high[0]) + 1).astype(str) + if ',' in dir_str: # type: ignore + return np.array( + dir_str.split(','), # type: ignore + dtype=int, + ).astype(str) # type: ignore + + if '--' in dir_str: # type: ignore + d_low, d_high = dir_str.split('--') # type: ignore + d_low = self._get_directions(d_low)[0] + d_high = self._get_directions(d_high)[0] + return np.arange(int(d_low), int(d_high) + 1).astype(str) # else: - raise ValueError( - f"Cannot parse direction string: " f"{dir_str}" - ) from exc + msg = f'Cannot parse direction string: ' f'{dir_str}' + raise ValueError(msg) from exc + + def query_error_setup(self, path: str) -> str | bool: + """ + Obtain error setup settings. + + Obtain settings associated with the treatment of errors and + warnings. + + Parameters + ---------- + path: str + Path to a setting. + + Returns + ------- + str | bool + Value of the setting + + Raises + ------ + KeyError + If the path does not point to a setting + + """ + error_setup = self._asmnt.options.error_setup + value = base.get(error_setup, path) + if value is None: + raise KeyError + return value diff --git a/pelicun/pelicun_warnings.py b/pelicun/pelicun_warnings.py new file mode 100644 index 000000000..399f32ab7 --- /dev/null +++ b/pelicun/pelicun_warnings.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . + +"""Pelicun warning and error classes.""" + +from __future__ import annotations + + +class PelicunWarning(Warning): + """Custom warning for specific use in the Pelicun project.""" + + +class PelicunInvalidConfigError(Exception): + """ + Exception raised for errors in the configuration of Pelicun. + + Attributes + ---------- + message: str + Explanation of the error. + + """ + + def __init__( + self, message: str = 'Invalid options in configuration file.' + ) -> None: + """Instantiate the error.""" + self.message = message + super().__init__(self.message) + + +class InconsistentUnitsError(Exception): + """ + Exception raised for inconsistent or invalid units. + + Attributes + ---------- + message : str + Explanation of the error. + """ + + def __init__( + self, message: str = 'Inconsistent units.', file: str | None = None + ) -> None: + self.message: str + + if file: + self.message = f'{self.message}\n' f'File: {file}' + else: + self.message = message + super().__init__(self.message) diff --git a/pelicun/resources/DamageAndLossModelLibrary b/pelicun/resources/DamageAndLossModelLibrary new file mode 160000 index 000000000..58de43688 --- /dev/null +++ b/pelicun/resources/DamageAndLossModelLibrary @@ -0,0 +1 @@ +Subproject commit 58de4368892feb3ff3369eb6e867bc681f086e11 diff --git a/pelicun/resources/SimCenterDBDL/damage_DB_Hazus_EQ_trnsp.csv b/pelicun/resources/SimCenterDBDL/damage_DB_Hazus_EQ_trnsp.csv index 4a365ad83..3e144a06a 100644 --- a/pelicun/resources/SimCenterDBDL/damage_DB_Hazus_EQ_trnsp.csv +++ b/pelicun/resources/SimCenterDBDL/damage_DB_Hazus_EQ_trnsp.csv @@ -29,7 +29,7 @@ HWB.GS.25,0,Spectral Acceleration|1.0,g,0,0,lognormal,0.3,0.6,,lognormal,0.5,0.6 HWB.GS.26,0,Spectral Acceleration|1.0,g,0,0,lognormal,0.75,0.6,,lognormal,0.75,0.6,,lognormal,0.75,0.6,,lognormal,1.1,0.6, HWB.GS.27,0,Spectral Acceleration|1.0,g,0,0,lognormal,0.75,0.6,,lognormal,0.75,0.6,,lognormal,0.75,0.6,,lognormal,1.1,0.6, HWB.GS.28,0,Spectral Acceleration|1.0,g,0,0,lognormal,0.8,0.6,,lognormal,1,0.6,,lognormal,1.2,0.6,,lognormal,1.7,0.6, -HWB.GF,0,Permanent Ground Deformation,inch,0,0,lognormal,3.9,0.2,,lognormal,13.8,0.2,,,,,,,,, +HWB.GF,0,Permanent Ground Deformation,inch,0,0,lognormal,3.9,0.2,,lognormal,3.9,0.2,,lognormal,3.9,0.2,,lognormal,13.8,0.2, HTU.GS.1,0,Peak Ground Acceleration,g,0,0,lognormal,0.6,0.6,,lognormal,0.8,0.6,,,,,,,,, HTU.GS.2,0,Peak Ground Acceleration,g,0,0,lognormal,0.5,0.6,,lognormal,0.7,0.6,,,,,,,,, -HTU.GF,0,Permanent Ground Deformation,inch,0,0,lognormal,6,0.7,,lognormal,12,0.5,,lognormal,60,0.5,,,,, +HTU.GF,0,Permanent Ground Deformation,inch,0,0,lognormal,6,0.7,,lognormal,12,0.5,,lognormal,60,0.5,,,,, \ No newline at end of file diff --git a/pelicun/resources/SimCenterDBDL/new_locations.json b/pelicun/resources/SimCenterDBDL/new_locations.json new file mode 100644 index 000000000..e0711f11d --- /dev/null +++ b/pelicun/resources/SimCenterDBDL/new_locations.json @@ -0,0 +1,20 @@ +{ + "damage_DB_FEMA_P58_2nd.csv": "seismic/building/component/FEMA P-58 2nd Edition/fragility.csv", + "damage_DB_FEMA_P58_2nd.json": "seismic/building/component/FEMA P-58 2nd Edition/fragility.json", + "damage_DB_Hazus_EQ_bldg.csv": "seismic/building/portfolio/Hazus v5.1/fragility.csv", + "damage_DB_Hazus_EQ_bldg.json": "seismic/building/portfolio/Hazus v5.1/fragility.json", + "damage_DB_Hazus_EQ_trnsp.csv": "seismic/transportation_network/portfolio/Hazus v5.1/fragility.csv", + "damage_DB_Hazus_EQ_trnsp.json": "seismic/transportation_network/portfolio/Hazus v5.1/fragility.json", + "damage_DB_Hazus_EQ_water.csv": "seismic/water_network/portfolio/fragility.csv", + "damage_DB_SimCenter_HU.csv": "hurricane/building/component/fragility.csv", + "damage_DB_SimCenter_HU.json": "hurricane/building/component/fragility.json", + "damage_DB_SimCenter_Hazus_HU_bldg.csv": "hurricane/building/portfolio/Hazus v4.2/fragility_fitted.csv", + "loss_repair_DB_FEMA_P58_2nd.csv": "seismic/building/component/FEMA P-58 2nd Edition/consequence_repair.csv", + "loss_repair_DB_FEMA_P58_2nd.json": "seismic/building/component/FEMA P-58 2nd Edition/consequence_repair.json", + "loss_repair_DB_Hazus_EQ_bldg.csv": "seismic/building/portfolio/Hazus v5.1/consequence_repair.csv", + "loss_repair_DB_Hazus_EQ_bldg.json": "seismic/building/portfolio/Hazus v5.1/consequence_repair.json", + "loss_repair_DB_Hazus_EQ_trnsp.csv": "seismic/transportation_network/portfolio/Hazus v5.1/consequence_repair.csv", + "loss_repair_DB_Hazus_EQ_trnsp.json": "seismic/transportation_network/portfolio/Hazus v5.1/consequence_repair.json", + "loss_repair_DB_SimCenter_Hazus_HU_bldg.csv": "hurricane/building/portfolio/Hazus v4.2/consequence_repair_fitted.csv", + "combined_loss_matrices/Wind_Flood_Hazus_HU_bldg.csv": "hurricane/building/portfolio/Hazus v4.2/combined_loss_matrices/Wind_Flood.csv" +} diff --git a/pelicun/resources/auto/Hazus_Earthquake_CSM.py b/pelicun/resources/auto/Hazus_Earthquake_CSM.py new file mode 100644 index 000000000..4c3b33de7 --- /dev/null +++ b/pelicun/resources/auto/Hazus_Earthquake_CSM.py @@ -0,0 +1,292 @@ +# +# Copyright (c) 2023 Leland Stanford Junior University +# Copyright (c) 2023 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . +# +# Contributors: +# Adam Zsarnóczay + +import pandas as pd + +ap_DesignLevel = {1940: 'LC', 1975: 'MC', 2100: 'HC'} +# ap_DesignLevel = {1940: 'PC', 1940: 'LC', 1975: 'MC', 2100: 'HC'} + +ap_DesignLevel_W1 = {0: 'LC', 1975: 'MC', 2100: 'HC'} +# ap_DesignLevel_W1 = {0: 'PC', 0: 'LC', 1975: 'MC', 2100: 'HC'} + +ap_Occupancy = { + 'Other/Unknown': 'RES3', + 'Residential - Single-Family': 'RES1', + 'Residential - Town-Home': 'RES3', + 'Residential - Multi-Family': 'RES3', + 'Residential - Mixed Use': 'RES3', + 'Office': 'COM4', + 'Hotel': 'RES4', + 'School': 'EDU1', + 'Industrial - Light': 'IND2', + 'Industrial - Warehouse': 'IND2', + 'Industrial - Heavy': 'IND1', + 'Retail': 'COM1', + 'Parking': 'COM10', +} + +convert_design_level = { + 'High-Code': 'HC', + 'Moderate-Code': 'MC', + 'Low-Code': 'LC', + 'Pre-Code': 'PC', +} + + +def convert_story_rise(structureType, stories): + if structureType in ['W1', 'W2', 'S3', 'PC1', 'MH']: + # These archetypes have no rise information in their IDs + rise = None + + else: + rise = None + # First, check if we have valid story information + try: + stories = int(stories) + + except (ValueError, TypeError) as exc: + msg = ( + 'Missing "NumberOfStories" information, ' + 'cannot infer `rise` attribute of archetype' + ) + + raise ValueError(msg) from exc + + if structureType == 'RM1': + if stories <= 3: + rise = 'L' + + else: + rise = 'M' + + elif structureType == 'URM': + if stories <= 2: + rise = 'L' + + else: + rise = 'M' + + elif structureType in [ + 'S1', + 'S2', + 'S4', + 'S5', + 'C1', + 'C2', + 'C3', + 'PC2', + 'RM2', + ]: + if stories <= 3: + rise = 'L' + + elif stories <= 7: + rise = 'M' + + else: + rise = 'H' + + return rise + + +def auto_populate(aim): + """ + Automatically creates a performance model for story EDP-based Hazus EQ analysis. + + Parameters + ---------- + aim: dict + Asset Information Model - provides features of the asset that can be + used to infer attributes of the performance model. + + Returns + ------- + gi_ap: dict + Extended General Information - extends the GI from the input AIM with + additional inferred features. These features are typically used in + intermediate steps during the auto-population and are not required + for the performance assessment. They are returned to allow reviewing + how these latent variables affect the final results. + dl_ap: dict + Damage and Loss parameters - these define the performance model and + details of the calculation. + comp: DataFrame + Component assignment - Defines the components (in rows) and their + location, direction, and quantity (in columns). + """ + + # extract the General Information + gi = aim.get('GeneralInformation', None) + + if gi is None: + # TODO: show an error message + pass + + # initialize the auto-populated gi + gi_ap = gi.copy() + + assetType = aim['assetType'] + ground_failure = aim['Applications']['DL']['ApplicationData']['ground_failure'] + + if assetType == 'Buildings': + # get the building parameters + bt = gi['StructureType'] # building type + + # get the design level + dl = gi.get('DesignLevel', None) + + if dl is None: + # If there is no DesignLevel provided, we assume that the YearBuilt is + # available + year_built = gi['YearBuilt'] + + if 'W1' in bt: + DesignL = ap_DesignLevel_W1 + else: + DesignL = ap_DesignLevel + + for year in sorted(DesignL.keys()): + if year_built <= year: + dl = DesignL[year] + break + + gi_ap['DesignLevel'] = dl + # get the number of stories / height + stories = gi.get('NumberOfStories', None) + + # We assume that the structure type does not include height information + # and we append it here based on the number of story information + rise = convert_story_rise(bt, stories) + + # get the number of stories / height + stories = gi.get('NumberOfStories', None) + + if rise is None: + # To prevent STR.W2.None.LC + fg_s = f'STR.{bt}.{dl}' + else: + fg_s = f'STR.{bt}.{rise}.{dl}' + # fg_s = f"STR.{bt}.{dl}" + fg_nsd = 'NSD' + fg_nsa = f'NSA.{dl}' + + comp = pd.DataFrame( + { + f'{fg_s}': [ + 'ea', + 1, + 1, + 1, + 'N/A', + ], + f'{fg_nsa}': [ + 'ea', + 1, + 0, + 1, + 'N/A', + ], + f'{fg_nsd}': [ + 'ea', + 1, + 1, + 1, + 'N/A', + ], + }, + index=['Units', 'Location', 'Direction', 'Theta_0', 'Family'], + ).T + + # if needed, add components to simulate damage from ground failure + if ground_failure: + foundation_type = 'S' + + # fmt: off + FG_GF_H = f'GF.H.{foundation_type}' + FG_GF_V = f'GF.V.{foundation_type}' + comp_gf = pd.DataFrame( + {f'{FG_GF_H}': [ 'ea', 1, 1, 1, 'N/A'], # noqa: E201, E241 + f'{FG_GF_V}': [ 'ea', 1, 3, 1, 'N/A']}, # noqa: E201, E241 + index = [ 'Units', 'Location', 'Direction', 'Theta_0', 'Family'] # noqa: E201, E251 + ).T + # fmt: on + + comp = pd.concat([comp, comp_gf], axis=0) + + # get the occupancy class + if gi['OccupancyClass'] in ap_Occupancy: + occupancy = ap_Occupancy[gi['OccupancyClass']] + else: + occupancy = gi['OccupancyClass'] + + plan_area = gi.get('PlanArea', 1.0) + + repair_config = { + 'ConsequenceDatabase': 'Hazus Earthquake - Buildings', + 'MapApproach': 'Automatic', + 'DecisionVariables': { + 'Cost': True, + 'Carbon': False, + 'Energy': False, + 'Time': False, + }, + } + + dl_ap = { + 'Asset': { + 'ComponentAssignmentFile': 'CMP_QNT.csv', + 'ComponentDatabase': 'Hazus Earthquake - Buildings', + 'NumberOfStories': f'{stories}', + 'OccupancyType': f'{occupancy}', + 'PlanArea': str(plan_area), + }, + 'Damage': {'DamageProcess': 'Hazus Earthquake'}, + 'Demands': {}, + 'Losses': {'Repair': repair_config}, + 'Options': { + 'NonDirectionalMultipliers': {'ALL': 1.0}, + }, + } + + else: + print( + f'AssetType: {assetType} is not supported ' + f'in Hazus Earthquake Capacity Spectrum Method-based DL method' + ) + + return gi_ap, dl_ap, comp diff --git a/pelicun/resources/auto/Hazus_Earthquake_IM.py b/pelicun/resources/auto/Hazus_Earthquake_IM.py index a2ea3f929..2712cfc85 100644 --- a/pelicun/resources/auto/Hazus_Earthquake_IM.py +++ b/pelicun/resources/auto/Hazus_Earthquake_IM.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2023 Leland Stanford Junior University # Copyright (c) 2023 The Regents of the University of California @@ -36,23 +35,31 @@ # # Contributors: # Adam Zsarnóczay +"""Hazus Earthquake IM.""" + from __future__ import annotations + import json +from pathlib import Path + +import numpy as np import pandas as pd + import pelicun +from pelicun.assessment import DLCalculationAssessment -ap_DesignLevel = {1940: 'LC', 1975: 'MC', 2100: 'HC'} +ap_design_level = {1940: 'LC', 1975: 'MC', 2100: 'HC'} # original: # ap_DesignLevel = {1940: 'PC', 1940: 'LC', 1975: 'MC', 2100: 'HC'} # Note that the duplicated key is ignored, and Python keeps the last # entry. -ap_DesignLevel_W1 = {0: 'LC', 1975: 'MC', 2100: 'HC'} +ap_design_level_w1 = {0: 'LC', 1975: 'MC', 2100: 'HC'} # original: # ap_DesignLevel_W1 = {0: 'PC', 0: 'LC', 1975: 'MC', 2100: 'HC'} # same thing applies -ap_Occupancy = { +ap_occupancy = { 'Other/Unknown': 'RES3', 'Residential - Single-Family': 'RES1', 'Residential - Town-Home': 'RES3', @@ -71,6 +78,9 @@ # Convert common length units def convertUnits(value, unit_in, unit_out): + """ + Convert units. + """ aval_types = ['m', 'mm', 'cm', 'km', 'inch', 'ft', 'mile'] m = 1.0 mm = 0.001 * m @@ -90,117 +100,170 @@ def convertUnits(value, unit_in, unit_out): } if (unit_in not in aval_types) or (unit_out not in aval_types): print( - f"The unit {unit_in} or {unit_out} " - f"are used in auto_population but not supported" + f'The unit {unit_in} or {unit_out} ' + f'are used in auto_population but not supported' ) - return - value = value * scale_map[unit_in] / scale_map[unit_out] - return value + return None + return value * scale_map[unit_in] / scale_map[unit_out] + + +def getHAZUSBridgeK3DModifier(hazus_class, aim): + # In HAZUS, the K_3D for HWB28 is undefined, so we return 1, i.e., no scaling + # The K-3D factors for HWB3 and HWB4 are defined as EQ1, which leads to division by zero + # This is an error in the HAZUS documentation, and we assume that the factors are 1 for these classes + mapping = { + 'HWB1': 1, + 'HWB2': 1, + 'HWB3': 1, + 'HWB4': 1, + 'HWB5': 1, + 'HWB6': 1, + 'HWB7': 1, + 'HWB8': 2, + 'HWB9': 3, + 'HWB10': 2, + 'HWB11': 3, + 'HWB12': 4, + 'HWB13': 4, + 'HWB14': 1, + 'HWB15': 5, + 'HWB16': 3, + 'HWB17': 1, + 'HWB18': 1, + 'HWB19': 1, + 'HWB20': 2, + 'HWB21': 3, + 'HWB22': 2, + 'HWB23': 3, + 'HWB24': 6, + 'HWB25': 6, + 'HWB26': 7, + 'HWB27': 7, + 'HWB28': 8, + } + factors = { + 1: (0.25, 1), + 2: (0.33, 0), + 3: (0.33, 1), + 4: (0.09, 1), + 5: (0.05, 0), + 6: (0.2, 1), + 7: (0.1, 0), + } + if hazus_class in ['HWB3', 'HWB4', 'HWB28']: + return 1 + else: + n = aim['NumOfSpans'] + a = factors[mapping[hazus_class]][0] + b = factors[mapping[hazus_class]][1] + return 1 + a / ( + n - b + ) # This is the original form in Mander and Basoz (1999) -def convertBridgeToHAZUSclass(AIM): +def convertBridgeToHAZUSclass(aim): # noqa: C901 # TODO: replace labels in AIM with standard CamelCase versions - structureType = AIM["BridgeClass"] + structure_type = aim['BridgeClass'] # if ( - # type(structureType) == str - # and len(structureType) > 3 - # and structureType[:3] == "HWB" - # and 0 < int(structureType[3:]) - # and 29 > int(structureType[3:]) + # type(structure_type) == str + # and len(structure_type) > 3 + # and structure_type[:3] == "HWB" + # and 0 < int(structure_type[3:]) + # and 29 > int(structure_type[3:]) # ): # return AIM["bridge_class"] - state = AIM["StateCode"] - yr_built = AIM["YearBuilt"] - num_span = AIM["NumOfSpans"] - len_max_span = AIM["MaxSpanLength"] - len_unit = AIM["units"]["length"] - len_max_span = convertUnits(len_max_span, len_unit, "m") + state = aim['StateCode'] + yr_built = aim['YearBuilt'] + num_span = aim['NumOfSpans'] + len_max_span = aim['MaxSpanLength'] + len_unit = aim['units']['length'] + len_max_span = convertUnits(len_max_span, len_unit, 'm') seismic = (int(state) == 6 and int(yr_built) >= 1975) or ( int(state) != 6 and int(yr_built) >= 1990 ) # Use a catch-all, other class by default - bridge_class = "HWB28" + bridge_class = 'HWB28' if len_max_span > 150: if not seismic: - bridge_class = "HWB1" + bridge_class = 'HWB1' else: - bridge_class = "HWB2" + bridge_class = 'HWB2' elif num_span == 1: if not seismic: - bridge_class = "HWB3" + bridge_class = 'HWB3' else: - bridge_class = "HWB4" + bridge_class = 'HWB4' - elif structureType in list(range(101, 107)): + elif structure_type in list(range(101, 107)): if not seismic: if state != 6: - bridge_class = "HWB5" + bridge_class = 'HWB5' else: - bridge_class = "HWB6" + bridge_class = 'HWB6' else: - bridge_class = "HWB7" + bridge_class = 'HWB7' - elif structureType in [205, 206]: + elif structure_type in [205, 206]: if not seismic: - bridge_class = "HWB8" + bridge_class = 'HWB8' else: - bridge_class = "HWB9" + bridge_class = 'HWB9' - elif structureType in list(range(201, 207)): + elif structure_type in list(range(201, 207)): if not seismic: - bridge_class = "HWB10" + bridge_class = 'HWB10' else: - bridge_class = "HWB11" + bridge_class = 'HWB11' - elif structureType in list(range(301, 307)): + elif structure_type in list(range(301, 307)): if not seismic: if len_max_span >= 20: if state != 6: - bridge_class = "HWB12" + bridge_class = 'HWB12' else: - bridge_class = "HWB13" + bridge_class = 'HWB13' else: if state != 6: - bridge_class = "HWB24" + bridge_class = 'HWB24' else: - bridge_class = "HWB25" + bridge_class = 'HWB25' else: - bridge_class = "HWB14" + bridge_class = 'HWB14' - elif structureType in list(range(402, 411)): + elif structure_type in list(range(402, 411)): if not seismic: if len_max_span >= 20: - bridge_class = "HWB15" + bridge_class = 'HWB15' elif state != 6: - bridge_class = "HWB26" + bridge_class = 'HWB26' else: - bridge_class = "HWB27" + bridge_class = 'HWB27' else: - bridge_class = "HWB16" + bridge_class = 'HWB16' - elif structureType in list(range(501, 507)): + elif structure_type in list(range(501, 507)): if not seismic: if state != 6: - bridge_class = "HWB17" + bridge_class = 'HWB17' else: - bridge_class = "HWB18" + bridge_class = 'HWB18' else: - bridge_class = "HWB19" + bridge_class = 'HWB19' - elif structureType in [605, 606]: + elif structure_type in [605, 606]: if not seismic: - bridge_class = "HWB20" + bridge_class = 'HWB20' else: - bridge_class = "HWB21" + bridge_class = 'HWB21' - elif structureType in list(range(601, 608)): + elif structure_type in list(range(601, 608)): if not seismic: - bridge_class = "HWB22" + bridge_class = 'HWB22' else: - bridge_class = "HWB23" + bridge_class = 'HWB23' # TODO: review and add HWB24-27 rules # TODO: also double check rules for HWB10-11 and HWB22-23 @@ -208,59 +271,98 @@ def convertBridgeToHAZUSclass(AIM): return bridge_class -def convertTunnelToHAZUSclass(AIM): - if ("Bored" in AIM["ConstructType"]) or ("Drilled" in AIM["ConstructType"]): - return "HTU1" - elif ("Cut" in AIM["ConstructType"]) or ("Cover" in AIM["ConstructType"]): - return "HTU2" +def getHAZUSBridgePGDModifier(hazus_class, aim): + # This is the original modifier in HAZUS, which gives inf if Skew is 0 + # modifier1 = 0.5*AIM['StructureLength']/(AIM['DeckWidth']*AIM['NumOfSpans']*np.sin(AIM['Skew']/180.0*np.pi)) + # Use the modifier that is corrected from HAZUS manual to achieve the asymptotic behavior + # Where longer bridges, narrower bridges, less span and higher skew leads to lower modifier (i.e., more fragile bridges) + modifier1 = ( + aim['DeckWidth'] + * aim['NumOfSpans'] + * np.sin((90 - aim['Skew']) / 180.0 * np.pi) + / (aim['StructureLength'] * 0.5) + ) + modifier2 = np.sin((90 - aim['Skew']) / 180.0 * np.pi) + mapping = { + 'HWB1': (1, 1), + 'HWB2': (1, 1), + 'HWB3': (1, 1), + 'HWB4': (1, 1), + 'HWB5': (modifier1, modifier1), + 'HWB6': (modifier1, modifier1), + 'HWB7': (modifier1, modifier1), + 'HWB8': (1, modifier2), + 'HWB9': (1, modifier2), + 'HWB10': (1, modifier2), + 'HWB11': (1, modifier2), + 'HWB12': (modifier1, modifier1), + 'HWB13': (modifier1, modifier1), + 'HWB14': (modifier1, modifier1), + 'HWB15': (1, modifier2), + 'HWB16': (1, modifier2), + 'HWB17': (modifier1, modifier1), + 'HWB18': (modifier1, modifier1), + 'HWB19': (modifier1, modifier1), + 'HWB20': (1, modifier2), + 'HWB21': (1, modifier2), + 'HWB22': (modifier1, modifier1), + 'HWB23': (modifier1, modifier1), + 'HWB24': (modifier1, modifier1), + 'HWB25': (modifier1, modifier1), + 'HWB26': (1, modifier2), + 'HWB27': (1, modifier2), + 'HWB28': (1, 1), + } + return mapping[hazus_class][0], mapping[hazus_class][1] + + +def convertTunnelToHAZUSclass(aim) -> str: + if ('Bored' in aim['ConstructType']) or ('Drilled' in aim['ConstructType']): + return 'HTU1' + elif ('Cut' in aim['ConstructType']) or ('Cover' in aim['ConstructType']): + return 'HTU2' else: # Select HTU2 for unclassified tunnels because it is more conservative. - return "HTU2" + return 'HTU2' -def convertRoadToHAZUSclass(AIM): - if AIM["RoadType"] in ["Primary", "Secondary"]: - return "HRD1" +def convertRoadToHAZUSclass(aim) -> str: + if aim['RoadType'] in ['Primary', 'Secondary']: + return 'HRD1' - elif AIM["RoadType"] == "Residential": - return "HRD2" + elif aim['RoadType'] == 'Residential': + return 'HRD2' else: # many unclassified roads are urban roads - return "HRD2" + return 'HRD2' -def convert_story_rise(structureType, stories): - if structureType in ['W1', 'W2', 'S3', 'PC1', 'MH']: +def convert_story_rise(structure_type, stories): + if structure_type in ['W1', 'W2', 'S3', 'PC1', 'MH']: # These archetypes have no rise information in their IDs rise = None else: + rise = None # Default value # First, check if we have valid story information try: stories = int(stories) except (ValueError, TypeError): - raise ValueError( + msg = ( 'Missing "NumberOfStories" information, ' 'cannot infer `rise` attribute of archetype' ) + raise ValueError(msg) # noqa: B904 - if structureType == 'RM1': - if stories <= 3: - rise = "L" - - else: - rise = "M" - - elif structureType == 'URM': - if stories <= 2: - rise = "L" + if structure_type == 'RM1': + rise = 'L' if stories <= 3 else 'M' - else: - rise = "M" + elif structure_type == 'URM': + rise = 'L' if stories <= 2 else 'M' - elif structureType in [ + elif structure_type in [ 'S1', 'S2', 'S4', @@ -272,18 +374,86 @@ def convert_story_rise(structureType, stories): 'RM2', ]: if stories <= 3: - rise = "L" + rise = 'L' elif stories <= 7: - rise = "M" + rise = 'M' else: - rise = "H" + rise = 'H' return rise -def auto_populate(AIM): +def getHAZUSBridgeSlightDamageModifier(hazus_class, aim): + if hazus_class in [ + 'HWB1', + 'HWB2', + 'HWB5', + 'HWB6', + 'HWB7', + 'HWB8', + 'HWB9', + 'HWB12', + 'HWB13', + 'HWB14', + 'HWB17', + 'HWB18', + 'HWB19', + 'HWB20', + 'HWB21', + 'HWB24', + 'HWB25', + 'HWB28', + ]: + return None + demand_path = Path(aim['DL']['Demands']['DemandFilePath']).resolve() + sample_size = int(aim['DL']['Demands']['SampleSize']) + length_unit = aim['GeneralInformation']['units']['length'] + coupled_demands = aim['Applications']['DL']['ApplicationData']['coupled_EDP'] + assessment = DLCalculationAssessment(config_options=None) + assessment.calculate_demand( + demand_path=demand_path, + collapse_limits=None, + length_unit=length_unit, + demand_calibration=None, + sample_size=sample_size, + demand_cloning=None, + residual_drift_inference=None, + coupled_demands=coupled_demands, + ) + demand_sample, _ = assessment.demand.save_sample(save_units=True) + edp_types = demand_sample.columns.get_level_values(level='type') + if (edp_types == 'SA_0.3').sum() != 1: + msg = ( + 'The demand file does not contain the required EDP type SA_0.3' + ' or contains multiple instances of it.' + ) + raise ValueError(msg) + sa_0p3 = demand_sample.loc[ # noqa: PD011 + :, demand_sample.columns.get_level_values(level='type') == 'SA_0.3' + ].values.flatten() + if (edp_types == 'SA_1.0').sum() != 1: + msg = ( + 'The demand file does not contain the required EDP type SA_1.0' + ' or contains multiple instances of it.' + ) + raise ValueError(msg) + sa_1p0 = demand_sample.loc[ # noqa: PD011 + :, demand_sample.columns.get_level_values(level='type') == 'SA_1.0' + ].values.flatten() + + ratio = 2.5 * sa_1p0 / sa_0p3 + operation = [ + f'*{ratio[i]}' if ratio[i] <= 1.0 else '*1.0' for i in range(len(ratio)) + ] + + assert len(operation) == sample_size + + return operation + + +def auto_populate(aim): # noqa: C901 """ Automatically creates a performance model for PGA-based Hazus EQ analysis. @@ -310,252 +480,309 @@ def auto_populate(AIM): """ # extract the General Information - GI = AIM.get('GeneralInformation', None) + gi = aim.get('GeneralInformation', None) - if GI is None: + if gi is None: # TODO: show an error message pass # initialize the auto-populated GI - GI_ap = GI.copy() + gi_ap = gi.copy() - assetType = AIM["assetType"] - ground_failure = AIM["Applications"]["DL"]["ApplicationData"]["ground_failure"] + asset_type = aim['assetType'] + ground_failure = aim['Applications']['DL']['ApplicationData']['ground_failure'] - if assetType == "Buildings": + if asset_type == 'Buildings': # get the building parameters - bt = GI['StructureType'] # building type + bt = gi['StructureType'] # building type # get the design level - dl = GI.get('DesignLevel', None) + dl = gi.get('DesignLevel', None) if dl is None: # If there is no DesignLevel provided, we assume that the YearBuilt is # available - year_built = GI['YearBuilt'] + year_built = gi['YearBuilt'] - if 'W1' in bt: - DesignL = ap_DesignLevel_W1 - else: - DesignL = ap_DesignLevel + design_l = ap_design_level_w1 if 'W1' in bt else ap_design_level - for year in sorted(DesignL.keys()): + for year in sorted(design_l.keys()): if year_built <= year: - dl = DesignL[year] + dl = design_l[year] break - GI_ap['DesignLevel'] = dl + gi_ap['DesignLevel'] = dl # get the number of stories / height - stories = GI.get('NumberOfStories', None) + stories = gi.get('NumberOfStories', None) # We assume that the structure type does not include height information # and we append it here based on the number of story information rise = convert_story_rise(bt, stories) if rise is not None: - LF = f'LF.{bt}.{rise}.{dl}' - GI_ap['BuildingRise'] = rise + lf = f'LF.{bt}.{rise}.{dl}' + gi_ap['BuildingRise'] = rise else: - LF = f'LF.{bt}.{dl}' + lf = f'LF.{bt}.{dl}' # fmt: off - CMP = pd.DataFrame( # noqa - {f'{LF}': ['ea', 1, 1, 1, 'N/A']}, # noqa - index = ['Units','Location','Direction','Theta_0','Family'] # noqa - ).T # noqa + comp = pd.DataFrame( + {f'{lf}': ['ea', 1, 1, 1, 'N/A']}, # noqa: E241 + index = ['Units','Location','Direction','Theta_0','Family'] # noqa: E231, E251 + ).T # fmt: on # if needed, add components to simulate damage from ground failure if ground_failure: foundation_type = 'S' - FG_GF_H = f'GF.H.{foundation_type}' - FG_GF_V = f'GF.V.{foundation_type}' + fg_gf_h = f'GF.H.{foundation_type}' + fg_gf_v = f'GF.V.{foundation_type}' # fmt: off - CMP_GF = pd.DataFrame( # noqa - {f'{FG_GF_H}':[ 'ea', 1, 1, 1, 'N/A'], # noqa - f'{FG_GF_V}':[ 'ea', 1, 3, 1, 'N/A']}, # noqa - index = [ 'Units','Location','Direction','Theta_0','Family'] # noqa - ).T # noqa + comp_gf = pd.DataFrame( + {f'{fg_gf_h}':[ 'ea', 1, 1, 1, 'N/A'], # noqa: E201, E231, E241 + f'{fg_gf_v}':[ 'ea', 1, 3, 1, 'N/A']}, # noqa: E201, E231, E241 + index = [ 'Units','Location','Direction','Theta_0','Family'] # noqa: E201, E231, E251 + ).T # fmt: on - CMP = pd.concat([CMP, CMP_GF], axis=0) + comp = pd.concat([comp, comp_gf], axis=0) # set the number of stories to 1 # there is only one component in a building-level resolution stories = 1 # get the occupancy class - if GI['OccupancyClass'] in ap_Occupancy.keys(): - ot = ap_Occupancy[GI['OccupancyClass']] + if gi['OccupancyClass'] in ap_occupancy: + occupancy_type = ap_occupancy[gi['OccupancyClass']] else: - ot = GI['OccupancyClass'] - - DL_ap = { - "Asset": { - "ComponentAssignmentFile": "CMP_QNT.csv", - "ComponentDatabase": "Hazus Earthquake - Buildings", - "NumberOfStories": f"{stories}", - "OccupancyType": f"{ot}", - "PlanArea": "1", + occupancy_type = gi['OccupancyClass'] + + dl_ap = { + 'Asset': { + 'ComponentAssignmentFile': 'CMP_QNT.csv', + 'ComponentDatabase': 'Hazus Earthquake - Buildings', + 'NumberOfStories': f'{stories}', + 'OccupancyType': f'{occupancy_type}', + 'PlanArea': '1', }, - "Damage": {"DamageProcess": "Hazus Earthquake"}, - "Demands": {}, - "Losses": { - "Repair": { - "ConsequenceDatabase": "Hazus Earthquake - Buildings", - "MapApproach": "Automatic", + 'Damage': {'DamageProcess': 'Hazus Earthquake'}, + 'Demands': {}, + 'Losses': { + 'Repair': { + 'ConsequenceDatabase': 'Hazus Earthquake - Buildings', + 'MapApproach': 'Automatic', } }, - "Options": { - "NonDirectionalMultipliers": {"ALL": 1.0}, + 'Options': { + 'NonDirectionalMultipliers': {'ALL': 1.0}, }, } - elif assetType == "TransportationNetwork": - inf_type = GI["assetSubtype"] + elif asset_type == 'TransportationNetwork': + inf_type = gi['assetSubtype'] + + if inf_type == 'HwyBridge': + # If Skew is labeled as 99, it means there is a major variation in skews of substructure units. (Per NBI coding guide) + # Assume a number of 45 as the "average" skew for the bridge. + if gi['Skew'] == 99: + gi['Skew'] = 45 - if inf_type == "HwyBridge": # get the bridge class - bt = convertBridgeToHAZUSclass(GI) - GI_ap['BridgeHazusClass'] = bt + bt = convertBridgeToHAZUSclass(gi) + gi_ap['BridgeHazusClass'] = bt # fmt: off - CMP = pd.DataFrame( # noqa - {f'HWB.GS.{bt[3:]}': [ 'ea', 1, 1, 1, 'N/A'], # noqa - f'HWB.GF': [ 'ea', 1, 1, 1, 'N/A']}, # noqa - index = [ 'Units','Location','Direction','Theta_0','Family'] # noqa - ).T # noqa + comp = pd.DataFrame( + {f'HWB.GS.{bt[3:]}': [ 'ea', 1, 1, 1, 'N/A']}, # noqa: E201, E241 + index = [ 'Units', 'Location', 'Direction', 'Theta_0', 'Family'] # noqa: E201, E251 + ).T # fmt: on - DL_ap = { - "Asset": { - "ComponentAssignmentFile": "CMP_QNT.csv", - "ComponentDatabase": "Hazus Earthquake - Transportation", - "BridgeHazusClass": bt, - "PlanArea": "1", + # scaling_specification + k_skew = np.sqrt(np.sin((90 - gi['Skew']) * np.pi / 180.0)) + k_3d = getHAZUSBridgeK3DModifier(bt, gi) + k_shape = getHAZUSBridgeSlightDamageModifier(bt, aim) + scaling_specification = { + f'HWB.GS.{bt[3:]}-1-1': { + 'LS2': f'*{k_skew * k_3d}', + 'LS3': f'*{k_skew * k_3d}', + 'LS4': f'*{k_skew * k_3d}', + } + } + if k_shape is not None: + scaling_specification[f'HWB.GS.{bt[3:]}-1-1']['LS1'] = k_shape + # if needed, add components to simulate damage from ground failure + if ground_failure: + # fmt: off + comp_gf = pd.DataFrame( + {f'HWB.GF': [ 'ea', 1, 1, 1, 'N/A']}, # noqa: E201, E241, F541 + index = [ 'Units', 'Location', 'Direction', 'Theta_0', 'Family'] # noqa: E201, E251 + ).T + # fmt: on + + comp = pd.concat([comp, comp_gf], axis=0) + + f1, f2 = getHAZUSBridgePGDModifier(bt, gi) + + scaling_specification.update( + { + 'HWB.GF-1-1': { + 'LS2': f'*{f1}', + 'LS3': f'*{f1}', + 'LS4': f'*{f2}', + } + } + ) + + dl_ap = { + 'Asset': { + 'ComponentAssignmentFile': 'CMP_QNT.csv', + 'ComponentDatabase': 'Hazus Earthquake - Transportation', + 'BridgeHazusClass': bt, + 'PlanArea': '1', + }, + 'Damage': { + 'DamageProcess': 'Hazus Earthquake', + 'ScalingSpecification': scaling_specification, }, - "Damage": {"DamageProcess": "Hazus Earthquake"}, - "Demands": {}, - "Losses": { - "Repair": { - "ConsequenceDatabase": "Hazus Earthquake - Transportation", - "MapApproach": "Automatic", + 'Demands': {}, + 'Losses': { + 'Repair': { + 'ConsequenceDatabase': 'Hazus Earthquake - Transportation', + 'MapApproach': 'Automatic', } }, - "Options": { - "NonDirectionalMultipliers": {"ALL": 1.0}, + 'Options': { + 'NonDirectionalMultipliers': {'ALL': 1.0}, }, } - elif inf_type == "HwyTunnel": + elif inf_type == 'HwyTunnel': # get the tunnel class - tt = convertTunnelToHAZUSclass(GI) - GI_ap['TunnelHazusClass'] = tt + tt = convertTunnelToHAZUSclass(gi) + gi_ap['TunnelHazusClass'] = tt # fmt: off - CMP = pd.DataFrame( # noqa - {f'HTU.GS.{tt[3:]}': [ 'ea', 1, 1, 1, 'N/A'], # noqa - f'HTU.GF': [ 'ea', 1, 1, 1, 'N/A']}, # noqa - index = [ 'Units','Location','Direction','Theta_0','Family'] # noqa - ).T # noqa + comp = pd.DataFrame( + {f'HTU.GS.{tt[3:]}': [ 'ea', 1, 1, 1, 'N/A']}, # noqa: E201, E241 + index = [ 'Units','Location','Direction','Theta_0','Family'] # noqa: E201, E231, E251 + ).T # fmt: on - - DL_ap = { - "Asset": { - "ComponentAssignmentFile": "CMP_QNT.csv", - "ComponentDatabase": "Hazus Earthquake - Transportation", - "TunnelHazusClass": tt, - "PlanArea": "1", + # if needed, add components to simulate damage from ground failure + if ground_failure: + # fmt: off + comp_gf = pd.DataFrame( + {f'HTU.GF': [ 'ea', 1, 1, 1, 'N/A']}, # noqa: E201, E241, F541 + index = [ 'Units','Location','Direction','Theta_0','Family'] # noqa: E201, E231, E251 + ).T + # fmt: on + + comp = pd.concat([comp, comp_gf], axis=0) + + dl_ap = { + 'Asset': { + 'ComponentAssignmentFile': 'CMP_QNT.csv', + 'ComponentDatabase': 'Hazus Earthquake - Transportation', + 'TunnelHazusClass': tt, + 'PlanArea': '1', }, - "Damage": {"DamageProcess": "Hazus Earthquake"}, - "Demands": {}, - "Losses": { - "Repair": { - "ConsequenceDatabase": "Hazus Earthquake - Transportation", - "MapApproach": "Automatic", + 'Damage': {'DamageProcess': 'Hazus Earthquake'}, + 'Demands': {}, + 'Losses': { + 'Repair': { + 'ConsequenceDatabase': 'Hazus Earthquake - Transportation', + 'MapApproach': 'Automatic', } }, - "Options": { - "NonDirectionalMultipliers": {"ALL": 1.0}, + 'Options': { + 'NonDirectionalMultipliers': {'ALL': 1.0}, }, } - elif inf_type == "Roadway": + elif inf_type == 'Roadway': # get the road class - rt = convertRoadToHAZUSclass(GI) - GI_ap['RoadHazusClass'] = rt + rt = convertRoadToHAZUSclass(gi) + gi_ap['RoadHazusClass'] = rt # fmt: off - CMP = pd.DataFrame( # noqa - {f'HRD.GF.{rt[3:]}':[ 'ea', 1, 1, 1, 'N/A']}, # noqa - index = [ 'Units','Location','Direction','Theta_0','Family'] # noqa - ).T # noqa + comp = pd.DataFrame( + {}, + index = [ 'Units','Location','Direction','Theta_0','Family'] # noqa: E201, E231, E251 + ).T # fmt: on - DL_ap = { - "Asset": { - "ComponentAssignmentFile": "CMP_QNT.csv", - "ComponentDatabase": "Hazus Earthquake - Transportation", - "RoadHazusClass": rt, - "PlanArea": "1", + if ground_failure: + # fmt: off + comp_gf = pd.DataFrame( + {f'HRD.GF.{rt[3:]}':[ 'ea', 1, 1, 1, 'N/A']}, # noqa: E201, E231, E241 + index = [ 'Units','Location','Direction','Theta_0','Family'] # noqa: E201, E231, E251 + ).T + # fmt: on + + comp = pd.concat([comp, comp_gf], axis=0) + + dl_ap = { + 'Asset': { + 'ComponentAssignmentFile': 'CMP_QNT.csv', + 'ComponentDatabase': 'Hazus Earthquake - Transportation', + 'RoadHazusClass': rt, + 'PlanArea': '1', }, - "Damage": {"DamageProcess": "Hazus Earthquake"}, - "Demands": {}, - "Losses": { - "Repair": { - "ConsequenceDatabase": "Hazus Earthquake - Transportation", - "MapApproach": "Automatic", + 'Damage': {'DamageProcess': 'Hazus Earthquake'}, + 'Demands': {}, + 'Losses': { + 'Repair': { + 'ConsequenceDatabase': 'Hazus Earthquake - Transportation', + 'MapApproach': 'Automatic', } }, - "Options": { - "NonDirectionalMultipliers": {"ALL": 1.0}, + 'Options': { + 'NonDirectionalMultipliers': {'ALL': 1.0}, }, } else: - print("subtype not supported in HWY") + print('subtype not supported in HWY') - elif assetType == "WaterDistributionNetwork": + elif asset_type == 'WaterDistributionNetwork': pipe_material_map = { - "CI": "B", - "AC": "B", - "RCC": "B", - "DI": "D", - "PVC": "D", - "DS": "B", - "BS": "D", + 'CI': 'B', + 'AC': 'B', + 'RCC': 'B', + 'DI': 'D', + 'PVC': 'D', + 'DS': 'D', + 'BS': 'B', } # GI = AIM.get("GeneralInformation", None) # if GI==None: # initialize the auto-populated GI - wdn_element_type = GI_ap.get("type", "MISSING") - asset_name = GI_ap.get("AIM_id", None) + wdn_element_type = gi_ap.get('type', 'MISSING') + asset_name = gi_ap.get('AIM_id', None) - if wdn_element_type == "Pipe": - pipe_construction_year = GI_ap.get("year", None) - pipe_diameter = GI_ap.get("Diam", None) + if wdn_element_type == 'Pipe': + pipe_construction_year = gi_ap.get('year', None) + pipe_diameter = gi_ap.get('Diam', None) # diamaeter value is a fundamental part of hydraulic # performance assessment if pipe_diameter is None: - raise ValueError( - f"pipe diamater in asset type {assetType}, \ - asset id \"{asset_name}\" has no diameter \ - value." - ) + msg = f'pipe diameter in asset type {asset_type}, \ + asset id "{asset_name}" has no diameter \ + value.' + raise ValueError(msg) - pipe_length = GI_ap.get("Len", None) + pipe_length = gi_ap.get('Len', None) # length value is a fundamental part of hydraulic performance assessment if pipe_diameter is None: - raise ValueError( - f"pipe length in asset type {assetType}, \ - asset id \"{asset_name}\" has no diameter \ - value." - ) + msg = f'pipe length in asset type {asset_type}, \ + asset id "{asset_name}" has no diameter \ + value.' + raise ValueError(msg) - pipe_material = GI_ap.get("material", None) + pipe_material = gi_ap.get('material', None) # pipe material can be not available or named "missing" in # both case, pipe flexibility will be set to "missing" @@ -565,60 +792,62 @@ def auto_populate(AIM): missing, if the pipe is smaller than or equal to 20 inches, the material is Cast Iron (CI) otherwise the pipe material is steel. - If the material is steel (ST), either based on user specified - input or the assumption due to the lack of the user-input, the year - that the pipe is constructed define the flexibility status per HAZUS - instructions. If the pipe is built in 1935 or after, it is, the pipe - is Ductile Steel (DS), and otherwise it is Brittle Steel (BS). - If the pipe is missing construction year and is built by steel, - we assume consevatively that the pipe is brittle (i.e., BS) + If the material is steel (ST), either based on user + specified input or the assumption due to the lack of the + user-input, the year that the pipe is constructed define + the flexibility status per HAZUS instructions. If the pipe + is built in 1935 or after, it is, the pipe is Ductile + Steel (DS), and otherwise it is Brittle Steel (BS). + If the pipe is missing construction year and is built + by steel, we assume consevatively that the pipe is brittle + (i.e., BS) """ if pipe_material is None: if pipe_diameter > 20 * 0.0254: # 20 inches in meter print( - f"Asset {asset_name} is missing material. Material is\ - assumed to be Cast Iron" + f'Asset {asset_name} is missing material. Material is\ + assumed to be Cast Iron' ) - pipe_material = "CI" + pipe_material = 'CI' else: print( - f"Asset {asset_name} is missing material. Material is " - f"assumed to be Steel (ST)" + f'Asset {asset_name} is missing material. Material is ' + f'assumed to be Steel (ST)' ) - pipe_material = "ST" + pipe_material = 'ST' - if pipe_material == "ST": + if pipe_material == 'ST': if (pipe_construction_year is not None) and ( pipe_construction_year >= 1935 ): print( - f"Asset {asset_name} has material of \"ST\" is assumed to be\ - Ductile Steel" + f'Asset {asset_name} has material of "ST" is assumed to be\ + Ductile Steel' ) - pipe_material = "DS" + pipe_material = 'DS' else: print( f'Asset {asset_name} has material of "ST" is assumed to be ' f'Brittle Steel' ) - pipe_material = "BS" + pipe_material = 'BS' - pipe_flexibility = pipe_material_map.get(pipe_material, "missing") + pipe_flexibility = pipe_material_map.get(pipe_material, 'missing') - GI_ap["material flexibility"] = pipe_flexibility - GI_ap["material"] = pipe_material + gi_ap['material flexibility'] = pipe_flexibility + gi_ap['material'] = pipe_material # Pipes are broken into 20ft segments (rounding up) and # each segment is represented by an individual entry in - # the performance model, `CMP`. The damage capcity of each + # the performance model, `CMP`. The damage capacity of each # segment is assumed to be independent and driven by the # same EDP. We therefore replicate the EDP associated with - # the pipe to the various locations assgined to the + # the pipe to the various locations assigned to the # segments. # Determine number of segments - pipe_length_unit = GI_ap['units']['length'] + pipe_length_unit = gi_ap['units']['length'] pipe_length_feet = pelicun.base.convert_units( pipe_length, unit=pipe_length_unit, to_unit='ft', category='length' ) @@ -629,19 +858,16 @@ def auto_populate(AIM): else: # In all other cases, round up. num_segments = int(pipe_length_feet / reference_length) + 1 - if num_segments > 1: - location_string = f'1--{num_segments}' - else: - location_string = '1' + location_string = f'1--{num_segments}' if num_segments > 1 else '1' # Define performance model # fmt: off - CMP = pd.DataFrame( # noqa - {f'PWP.{pipe_flexibility}.GS': ['ea', location_string, '0', 1, 'N/A'], # noqa - f'PWP.{pipe_flexibility}.GF': ['ea', location_string, '0', 1, 'N/A'], # noqa - 'aggregate': ['ea', location_string, '0', 1, 'N/A']}, # noqa - index = ['Units','Location','Direction','Theta_0','Family'] # noqa - ).T # noqa + comp = pd.DataFrame( + {f'PWP.{pipe_flexibility}.GS': ['ea', location_string, '0', 1, 'N/A'], + f'PWP.{pipe_flexibility}.GF': ['ea', location_string, '0', 1, 'N/A'], + 'aggregate': ['ea', location_string, '0', 1, 'N/A']}, + index = ['Units','Location','Direction','Theta_0','Family'] # noqa: E231, E251 + ).T # fmt: on # Set up the demand cloning configuration for the pipe @@ -658,7 +884,7 @@ def auto_populate(AIM): ) demand_cloning_config = {} for edp in response_data.columns: - tag, location, direction = edp + tag, location, direction = edp # noqa: F841 demand_cloning_config['-'.join(edp)] = [ f'{tag}-{x}-{direction}' @@ -668,112 +894,109 @@ def auto_populate(AIM): # Create damage process dmg_process = { - f"1_PWP.{pipe_flexibility}.GS-LOC": {"DS1": "aggregate_DS1"}, - f"2_PWP.{pipe_flexibility}.GF-LOC": {"DS1": "aggregate_DS1"}, - f"3_PWP.{pipe_flexibility}.GS-LOC": {"DS2": "aggregate_DS2"}, - f"4_PWP.{pipe_flexibility}.GF-LOC": {"DS2": "aggregate_DS2"}, + f'1_PWP.{pipe_flexibility}.GS-LOC': {'DS1': 'aggregate_DS1'}, + f'2_PWP.{pipe_flexibility}.GF-LOC': {'DS1': 'aggregate_DS1'}, + f'3_PWP.{pipe_flexibility}.GS-LOC': {'DS2': 'aggregate_DS2'}, + f'4_PWP.{pipe_flexibility}.GF-LOC': {'DS2': 'aggregate_DS2'}, } dmg_process_filename = 'dmg_process.json' with open(dmg_process_filename, 'w', encoding='utf-8') as f: json.dump(dmg_process, f, indent=2) # Define the auto-populated config - DL_ap = { - "Asset": { - "ComponentAssignmentFile": "CMP_QNT.csv", - "ComponentDatabase": "Hazus Earthquake - Water", - "Material Flexibility": pipe_flexibility, - "PlanArea": "1", # Sina: does not make sense for water. + dl_ap = { + 'Asset': { + 'ComponentAssignmentFile': 'CMP_QNT.csv', + 'ComponentDatabase': 'Hazus Earthquake - Water', + 'Material Flexibility': pipe_flexibility, + 'PlanArea': '1', # Sina: does not make sense for water. # Kept it here since itw as also # kept here for Transportation }, - "Damage": { - "DamageProcess": "User Defined", - "DamageProcessFilePath": "dmg_process.json", + 'Damage': { + 'DamageProcess': 'User Defined', + 'DamageProcessFilePath': 'dmg_process.json', }, - "Demands": demand_config, + 'Demands': demand_config, } - elif wdn_element_type == "Tank": + elif wdn_element_type == 'Tank': tank_cmp_lines = { - ("OG", "C", 1): {'PST.G.C.A.GS': ['ea', 1, 1, 1, 'N/A']}, - ("OG", "C", 0): {'PST.G.C.U.GS': ['ea', 1, 1, 1, 'N/A']}, - ("OG", "S", 1): {'PST.G.S.A.GS': ['ea', 1, 1, 1, 'N/A']}, - ("OG", "S", 0): {'PST.G.S.U.GS': ['ea', 1, 1, 1, 'N/A']}, + ('OG', 'C', 1): {'PST.G.C.A.GS': ['ea', 1, 1, 1, 'N/A']}, + ('OG', 'C', 0): {'PST.G.C.U.GS': ['ea', 1, 1, 1, 'N/A']}, + ('OG', 'S', 1): {'PST.G.S.A.GS': ['ea', 1, 1, 1, 'N/A']}, + ('OG', 'S', 0): {'PST.G.S.U.GS': ['ea', 1, 1, 1, 'N/A']}, # Anchored status and Wood is not defined for On Ground tanks - ("OG", "W", 0): {'PST.G.W.GS': ['ea', 1, 1, 1, 'N/A']}, + ('OG', 'W', 0): {'PST.G.W.GS': ['ea', 1, 1, 1, 'N/A']}, # Anchored status and Steel is not defined for Above Ground tanks - ("AG", "S", 0): {'PST.A.S.GS': ['ea', 1, 1, 1, 'N/A']}, + ('AG', 'S', 0): {'PST.A.S.GS': ['ea', 1, 1, 1, 'N/A']}, # Anchored status and Concrete is not defined for Buried tanks. - ("B", "C", 0): {'PST.B.C.GF': ['ea', 1, 1, 1, 'N/A']}, + ('B', 'C', 0): {'PST.B.C.GF': ['ea', 1, 1, 1, 'N/A']}, } # The default values are assumed: material = Concrete (C), # location= On Ground (OG), and Anchored = 1 - tank_material = GI_ap.get("material", "C") - tank_location = GI_ap.get("location", "OG") - tank_anchored = GI_ap.get("anchored", int(1)) + tank_material = gi_ap.get('material', 'C') + tank_location = gi_ap.get('location', 'OG') + tank_anchored = gi_ap.get('anchored', 1) - tank_material_allowable = {"C", "S"} + tank_material_allowable = {'C', 'S'} if tank_material not in tank_material_allowable: - raise ValueError( - f"Tank's material = \"{tank_material}\" is \ + msg = f'Tank\'s material = "{tank_material}" is \ not allowable in tank {asset_name}. The \ material must be either C for concrete or S \ - for steel." - ) + for steel.' + raise ValueError(msg) - tank_location_allowable = {"AG", "OG", "B"} + tank_location_allowable = {'AG', 'OG', 'B'} if tank_location not in tank_location_allowable: - raise ValueError( - f"Tank's location = \"{tank_location}\" is \ + msg = f'Tank\'s location = "{tank_location}" is \ not allowable in tank {asset_name}. The \ - location must be either \"AG\" for Above \ - ground, \"OG\" for On Ground or \"BG\" for \ - Bellow Ground (burried) Tanks." - ) + location must be either "AG" for Above \ + ground, "OG" for On Ground or "BG" for \ + Below Ground (buried) Tanks.' + raise ValueError(msg) - tank_anchored_allowable = {int(0), int(1)} + tank_anchored_allowable = {0, 1} if tank_anchored not in tank_anchored_allowable: - raise ValueError( - f"Tank's anchored status = \"{tank_location}\ - \" is not allowable in tank {asset_name}. \ + msg = f'Tank\'s anchored status = "{tank_location}\ + " is not allowable in tank {asset_name}. \ The anchored status must be either integer\ - value 0 for unachored, or 1 for anchored" - ) + value 0 for unachored, or 1 for anchored' + raise ValueError(msg) - if tank_location == "AG" and tank_material == "C": + if tank_location == 'AG' and tank_material == 'C': print( - f"The tank {asset_name} is Above Ground (i.e., AG), but \ - the material type is Concrete (\"C\"). Tank type \"C\" is not \ - defiend for AG tanks. The tank is assumed to be Steel (\"S\")" + f'The tank {asset_name} is Above Ground (i.e., AG), but \ + the material type is Concrete ("C"). Tank type "C" is not \ + defined for AG tanks. The tank is assumed to be Steel ("S")' ) - tank_material = "S" + tank_material = 'S' - if tank_location == "AG" and tank_material == "W": + if tank_location == 'AG' and tank_material == 'W': print( - f"The tank {asset_name} is Above Ground (i.e., AG), but \ - the material type is Wood (\"W\"). Tank type \"W\" is not \ - defiend for AG tanks. The tank is assumed to be Steel (\"S\")" + f'The tank {asset_name} is Above Ground (i.e., AG), but \ + the material type is Wood ("W"). Tank type "W" is not \ + defined for AG tanks. The tank is assumed to be Steel ("S")' ) - tank_material = "S" + tank_material = 'S' - if tank_location == "B" and tank_material == "S": + if tank_location == 'B' and tank_material == 'S': print( - f"The tank {asset_name} is burried (i.e., B), but the\ - material type is Steel (\"S\"). \ - Tank type \"S\" is not defiend for\ - B tanks. The tank is assumed to be Concrete (\"C\")" + f'The tank {asset_name} is buried (i.e., B), but the\ + material type is Steel ("S"). \ + Tank type "S" is not defined for\ + B tanks. The tank is assumed to be Concrete ("C")' ) - tank_material = "C" + tank_material = 'C' - if tank_location == "B" and tank_material == "W": + if tank_location == 'B' and tank_material == 'W': print( - f"The tank {asset_name} is burried (i.e., B), but the\ - material type is Wood (\"W\"). Tank type \"W\" is not defiend \ - for B tanks. The tank is assumed to be Concrete (\"C\")" + f'The tank {asset_name} is buried (i.e., B), but the\ + material type is Wood ("W"). Tank type "W" is not defined \ + for B tanks. The tank is assumed to be Concrete ("C")' ) - tank_material = "C" + tank_material = 'C' if tank_anchored == 1: # Since anchore status does nto matter, there is no need to @@ -781,40 +1004,40 @@ def auto_populate(AIM): tank_anchored = 0 cur_tank_cmp_line = tank_cmp_lines[ - (tank_location, tank_material, tank_anchored) + tank_location, tank_material, tank_anchored ] - CMP = pd.DataFrame( + comp = pd.DataFrame( cur_tank_cmp_line, index=['Units', 'Location', 'Direction', 'Theta_0', 'Family'], ).T - DL_ap = { - "Asset": { - "ComponentAssignmentFile": "CMP_QNT.csv", - "ComponentDatabase": "Hazus Earthquake - Water", - "Material": tank_material, - "Location": tank_location, - "Anchored": tank_anchored, - "PlanArea": "1", # Sina: does not make sense for water. + dl_ap = { + 'Asset': { + 'ComponentAssignmentFile': 'CMP_QNT.csv', + 'ComponentDatabase': 'Hazus Earthquake - Water', + 'Material': tank_material, + 'Location': tank_location, + 'Anchored': tank_anchored, + 'PlanArea': '1', # Sina: does not make sense for water. # Kept it here since itw as also kept here for Transportation }, - "Damage": {"DamageProcess": "Hazus Earthquake"}, - "Demands": {}, + 'Damage': {'DamageProcess': 'Hazus Earthquake'}, + 'Demands': {}, } else: print( - f"Water Distribution network element type {wdn_element_type} " - f"is not supported in Hazus Earthquake IM DL method" + f'Water Distribution network element type {wdn_element_type} ' + f'is not supported in Hazus Earthquake IM DL method' ) - DL_ap = None - CMP = None + dl_ap = None + comp = None else: print( - f"AssetType: {assetType} is not supported " - f"in Hazus Earthquake IM DL method" + f'AssetType: {asset_type} is not supported ' + f'in Hazus Earthquake IM DL method' ) - return GI_ap, DL_ap, CMP + return gi_ap, dl_ap, comp diff --git a/pelicun/resources/auto/Hazus_Earthquake_Story.py b/pelicun/resources/auto/Hazus_Earthquake_Story.py index 4b29a3feb..e7f7597f5 100644 --- a/pelicun/resources/auto/Hazus_Earthquake_Story.py +++ b/pelicun/resources/auto/Hazus_Earthquake_Story.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2023 Leland Stanford Junior University # Copyright (c) 2023 The Regents of the University of California @@ -38,15 +37,16 @@ # Adam Zsarnóczay from __future__ import annotations + import pandas as pd -ap_DesignLevel = {1940: 'LC', 1975: 'MC', 2100: 'HC'} +ap_design_level = {1940: 'LC', 1975: 'MC', 2100: 'HC'} # ap_DesignLevel = {1940: 'PC', 1940: 'LC', 1975: 'MC', 2100: 'HC'} -ap_DesignLevel_W1 = {0: 'LC', 1975: 'MC', 2100: 'HC'} +ap_design_level_w1 = {0: 'LC', 1975: 'MC', 2100: 'HC'} # ap_DesignLevel_W1 = {0: 'PC', 0: 'LC', 1975: 'MC', 2100: 'HC'} -ap_Occupancy = { +ap_occupancy = { 'Other/Unknown': 'RES3', 'Residential - Single-Family': 'RES1', 'Residential - Town-Home': 'RES3', @@ -70,7 +70,7 @@ } -def story_scale(stories, comp_type): +def story_scale(stories, comp_type): # noqa: C901 if comp_type == 'NSA': if stories == 1: return 1.00 @@ -108,11 +108,7 @@ def story_scale(stories, comp_type): return 2.75 elif stories == 5: return 3.00 - elif stories == 6: - return 3.50 - elif stories == 7: - return 3.50 - elif stories == 8: + elif stories in (6, 7, 8): return 3.50 elif stories == 9: return 4.50 @@ -122,9 +118,10 @@ def story_scale(stories, comp_type): return 7.30 else: return 1.0 + return None -def auto_populate(AIM): +def auto_populate(aim): """ Automatically creates a performance model for story EDP-based Hazus EQ analysis. @@ -148,69 +145,66 @@ def auto_populate(AIM): CMP: DataFrame Component assignment - Defines the components (in rows) and their location, direction, and quantity (in columns). - """ + """ # extract the General Information - GI = AIM.get('GeneralInformation', None) + gi = aim.get('GeneralInformation', None) - if GI is None: + if gi is None: # TODO: show an error message pass # initialize the auto-populated GI - GI_ap = GI.copy() + gi_ap = gi.copy() - assetType = AIM["assetType"] - ground_failure = AIM["Applications"]["DL"]["ApplicationData"]["ground_failure"] + asset_type = aim['assetType'] + ground_failure = aim['Applications']['DL']['ApplicationData']['ground_failure'] - if assetType == "Buildings": + if asset_type == 'Buildings': # get the building parameters - bt = GI['StructureType'] # building type + bt = gi['StructureType'] # building type # get the design level - dl = GI.get('DesignLevel', None) + dl = gi.get('DesignLevel', None) if dl is None: # If there is no DesignLevel provided, we assume that the YearBuilt is # available - year_built = GI['YearBuilt'] + year_built = gi['YearBuilt'] - if 'W1' in bt: - DesignL = ap_DesignLevel_W1 - else: - DesignL = ap_DesignLevel + design_l = ap_design_level_w1 if 'W1' in bt else ap_design_level - for year in sorted(DesignL.keys()): + for year in sorted(design_l.keys()): if year_built <= year: - dl = DesignL[year] + dl = design_l[year] break - GI_ap['DesignLevel'] = dl + gi_ap['DesignLevel'] = dl # get the number of stories / height - stories = GI.get('NumberOfStories', None) + stories = gi.get('NumberOfStories', None) - FG_S = f'STR.{bt}.{dl}' - FG_NSD = 'NSD' - FG_NSA = f'NSA.{dl}' + fg_s = f'STR.{bt}.{dl}' + fg_nsd = 'NSD' + fg_nsa = f'NSA.{dl}' - CMP = pd.DataFrame( + comp = pd.DataFrame( { - f'{FG_S}': [ + f'{fg_s}': [ 'ea', 'all', '1, 2', f"{story_scale(stories, 'S') / stories / 2.}", 'N/A', ], - f'{FG_NSA}': [ + f'{fg_nsa}': [ 'ea', 'all', 0, f"{story_scale(stories, 'NSA') / stories}", 'N/A', ], - f'{FG_NSD}': [ + f'{fg_nsd}': [ 'ea', 'all', '1, 2', @@ -225,57 +219,57 @@ def auto_populate(AIM): if ground_failure: foundation_type = 'S' - # fmt: off - FG_GF_H = f'GF.H.{foundation_type}' # noqa - FG_GF_V = f'GF.V.{foundation_type}' # noqa - CMP_GF = pd.DataFrame( # noqa - {f'{FG_GF_H}':[ 'ea', 1, 1, 1, 'N/A'], # noqa - f'{FG_GF_V}':[ 'ea', 1, 3, 1, 'N/A']}, # noqa - index = [ 'Units','Location','Direction','Theta_0','Family'] # noqa - ).T # noqa - # fmt: on + FG_GF_H = f'GF.H.{foundation_type}' + FG_GF_V = f'GF.V.{foundation_type}' + CMP_GF = pd.DataFrame( + { + f'{FG_GF_H}': ['ea', 1, 1, 1, 'N/A'], + f'{FG_GF_V}': ['ea', 1, 3, 1, 'N/A'], + }, + index=['Units', 'Location', 'Direction', 'Theta_0', 'Family'], + ).T - CMP = pd.concat([CMP, CMP_GF], axis=0) + comp = pd.concat([comp, CMP_GF], axis=0) # get the occupancy class - if GI['OccupancyClass'] in ap_Occupancy.keys(): - ot = ap_Occupancy[GI['OccupancyClass']] + if gi['OccupancyClass'] in ap_occupancy: + occupancy_type = ap_occupancy[gi['OccupancyClass']] else: - ot = GI['OccupancyClass'] + occupancy_type = gi['OccupancyClass'] - plan_area = GI.get('PlanArea', 1.0) + plan_area = gi.get('PlanArea', 1.0) repair_config = { - "ConsequenceDatabase": "Hazus Earthquake - Stories", - "MapApproach": "Automatic", - "DecisionVariables": { - "Cost": True, - "Carbon": False, - "Energy": False, - "Time": False, + 'ConsequenceDatabase': 'Hazus Earthquake - Stories', + 'MapApproach': 'Automatic', + 'DecisionVariables': { + 'Cost': True, + 'Carbon': False, + 'Energy': False, + 'Time': False, }, } - DL_ap = { - "Asset": { - "ComponentAssignmentFile": "CMP_QNT.csv", - "ComponentDatabase": "Hazus Earthquake - Stories", - "NumberOfStories": f"{stories}", - "OccupancyType": f"{ot}", - "PlanArea": str(plan_area), + dl_ap = { + 'Asset': { + 'ComponentAssignmentFile': 'CMP_QNT.csv', + 'ComponentDatabase': 'Hazus Earthquake - Stories', + 'NumberOfStories': f'{stories}', + 'OccupancyType': f'{occupancy_type}', + 'PlanArea': str(plan_area), }, - "Damage": {"DamageProcess": "Hazus Earthquake"}, - "Demands": {}, - "Losses": {"Repair": repair_config}, - "Options": { - "NonDirectionalMultipliers": {"ALL": 1.0}, + 'Damage': {'DamageProcess': 'Hazus Earthquake'}, + 'Demands': {}, + 'Losses': {'Repair': repair_config}, + 'Options': { + 'NonDirectionalMultipliers': {'ALL': 1.0}, }, } else: print( - f"AssetType: {assetType} is not supported " - f"in Hazus Earthquake Story-based DL method" + f'AssetType: {asset_type} is not supported ' + f'in Hazus Earthquake Story-based DL method' ) - return GI_ap, DL_ap, CMP + return gi_ap, dl_ap, comp diff --git a/pelicun/settings/default_config.json b/pelicun/settings/default_config.json index 47fe005aa..1d9da5e5c 100644 --- a/pelicun/settings/default_config.json +++ b/pelicun/settings/default_config.json @@ -24,7 +24,14 @@ "SampleSize": 1000, "PreserveRawOrder": false }, - "RepairCostAndTimeCorrelation": 0.0 + "RepairCostAndTimeCorrelation": 0.0, + "ErrorSetup": { + "Loss": { + "ReplacementThreshold": { + "RaiseOnUnknownKeys": true + } + } + } }, "DemandAssessment": { "Calibration": { diff --git a/pelicun/settings/default_units.json b/pelicun/settings/default_units.json index fe9a3122f..1b387b4a4 100644 --- a/pelicun/settings/default_units.json +++ b/pelicun/settings/default_units.json @@ -37,7 +37,7 @@ "inchps": 0.0254, "ftps": 0.3048 }, - "accelleration": { + "acceleration": { "mps2": 1.0, "inps2": 0.0254, "inchps2": 0.0254, diff --git a/pelicun/settings/input_schema.json b/pelicun/settings/input_schema.json new file mode 100644 index 000000000..19f54ce6b --- /dev/null +++ b/pelicun/settings/input_schema.json @@ -0,0 +1,645 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "GeneralInformation": { + "type": "object", + "properties": { + "AssetName": { + "type": "string" + }, + "AssetType": { + "type": "string" + }, + "Location": { + "type": "object", + "properties": { + "Latitude": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "Longitude": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + } + } + }, + "units": { + "type": "object", + "properties": { + "length": { + "type": "string", + "examples": [ + "in", + "m" + ] + } + }, + "required": [ + "length" + ] + } + }, + "required": [ + "units" + ] + }, + "assetType": { + "type": "string", + "examples": [ + "Buildings" + ] + }, + "DL": { + "type": "object", + "properties": { + "Demands": { + "type": "object", + "properties": { + "DemandFilePath": { + "type": "string" + }, + "SampleSize": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ] + }, + "CoupledDemands": { + "type": "boolean" + }, + "Calibration": { + "type": "object" + }, + "CollapseLimits": { + "type": "object", + "patternProperties": { + ".*": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + } + } + }, + "InferResidualDrift": { + "type": "object", + "properties": { + "method": { + "type": "string" + }, + "x-direction": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "y-direction": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + } + }, + "required": [ + "method" + ] + } + }, + "required": [ + "DemandFilePath" + ] + }, + "Asset": { + "type": "object", + "properties": { + "ComponentAssignmentFile": { + "type": "string" + }, + "NumberOfStories": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "examples": [ + 1, + 5, + 10 + ] + }, + "ComponentSampleFile": { + "type": "string" + }, + "ComponentDatabase": { + "type": "string" + }, + "ComponentDatabasePath": { + "type": "string" + } + }, + "required": [ + "ComponentAssignmentFile" + ] + }, + "Damage": { + "type": "object", + "properties": { + "CollapseFragility": { + "type": "object", + "properties": { + "DemandType": { + "type": "string", + "examples": [ + "SA", + "PFA", + "PGA" + ] + }, + "CapacityDistribution": { + "type": "string" + }, + "CapacityMedian": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "Theta_1": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + } + }, + "required": [ + "DemandType", + "CapacityDistribution", + "CapacityMedian", + "Theta_1" + ] + }, + "DamageProcess": { + "type": "string" + }, + "IrreparableDamage": { + "type": "object", + "properties": { + "DriftCapacityMedian": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "DriftCapacityLogStd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + } + }, + "required": [ + "DriftCapacityMedian", + "DriftCapacityLogStd" + ] + } + }, + "required": [ + "DamageProcess" + ] + }, + "Losses": { + "type": "object", + "properties": { + "Repair": { + "type": "object", + "properties": { + "ConsequenceDatabase": { + "type": "string" + }, + "MapApproach": { + "type": "string" + }, + "MapFilePath": { + "type": "string" + }, + "DecisionVariables": { + "type": "object", + "properties": { + "Cost": { + "type": "boolean" + }, + "Time": { + "type": "boolean" + }, + "Carbon": { + "type": "boolean" + }, + "Energy": { + "type": "boolean" + } + } + }, + "ConsequenceDatabasePath": { + "type": "string" + }, + "ReplacementEnergy": { + "type": "object", + "properties": { + "Unit": { + "type": "string" + }, + "Median": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "Distribution": { + "type": "string" + }, + "Theta_1": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + } + }, + "required": [ + "Unit", + "Median", + "Distribution", + "Theta_1" + ] + }, + "ReplacementCarbon": { + "type": "object", + "properties": { + "Unit": { + "type": "string" + }, + "Median": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "Distribution": { + "type": "string" + }, + "Theta_1": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + } + }, + "required": [ + "Unit", + "Median", + "Distribution", + "Theta_1" + ] + }, + "ReplacementTime": { + "type": "object", + "properties": { + "Unit": { + "type": "string" + }, + "Median": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "Distribution": { + "type": "string" + }, + "Theta_1": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + } + }, + "required": [ + "Unit", + "Median", + "Distribution", + "Theta_1" + ] + }, + "ReplacementCost": { + "type": "object", + "properties": { + "Unit": { + "type": "string" + }, + "Median": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "Distribution": { + "type": "string" + }, + "Theta_1": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + } + }, + "required": [ + "Unit", + "Median", + "Distribution", + "Theta_1" + ] + } + } + } + } + }, + "Outputs": { + "type": "object", + "properties": { + "Demand": { + "type": "object", + "properties": { + "Sample": { + "type": "boolean" + }, + "Statistics": { + "type": "boolean" + } + } + }, + "Asset": { + "type": "object", + "properties": { + "Sample": { + "type": "boolean" + }, + "Statistics": { + "type": "boolean" + } + } + }, + "Damage": { + "type": "object", + "properties": { + "Sample": { + "type": "boolean" + }, + "Statistics": { + "type": "boolean" + }, + "GroupedSample": { + "type": "boolean" + }, + "GroupedStatistics": { + "type": "boolean" + } + } + }, + "Loss": { + "type": "object", + "properties": { + "Repair": { + "type": "object", + "properties": { + "Sample": { + "type": "boolean" + }, + "Statistics": { + "type": "boolean" + }, + "GroupedSample": { + "type": "boolean" + }, + "GroupedStatistics": { + "type": "boolean" + }, + "AggregateSample": { + "type": "boolean" + }, + "AggregateStatistics": { + "type": "boolean" + } + } + } + } + }, + "Format": { + "type": "object", + "properties": { + "CSV": { + "type": "boolean" + }, + "JSON": { + "type": "boolean" + } + } + }, + "Settings": { + "type": "object", + "properties": { + "CondenseDS": { + "type": "boolean" + }, + "SimpleIndexInJSON": { + "type": "boolean" + }, + "AggregateColocatedComponentResults": { + "type": "boolean" + } + } + } + } + }, + "Options": { + "type": "object", + "properties": { + "Options": { + "type": "boolean" + }, + "Seed": { + "type": "integer" + }, + "LogShowMS": { + "type": "boolean" + }, + "LogFile": { + "type": "string" + }, + "UnitsFile": { + "type": "string" + }, + "PrintLog": { + "type": "boolean" + }, + "ShowWarnings": { + "type": "boolean" + }, + "DemandOffset": { + "type": "object" + }, + "ListAllDamageStates": { + "type": "boolean" + }, + "NonDirectionalMultipliers": { + "type": "object" + }, + "EconomiesOfScale": { + "type": "object", + "properties": { + "AcrossFlorrs": { + "type": "boolean" + }, + "AcrossDamageStates": { + "type": "boolean" + } + } + }, + "Sampling": { + "type": "object", + "properties": { + "SamplingMethod": { + "type": "string" + }, + "SampleSize": { + "type": "integer" + }, + "PreserveRawOrder": { + "type": "boolean" + } + } + }, + "RepairCostAndTimeCorrelation": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + } + } + }, + "DemandAssessment": { + "type": "object", + "properties": { + "Calibration": { + "type": "object", + "properties": { + "Marginals": { + "type": "object" + } + } + } + } + }, + "ApplicationData": { + "type": "object", + "properties": { + "ground_failure": { + "type": "boolean" + } + } + } + } + }, + "Applications": { + "type": "object" + }, + "auto_script_path": { + "type": "string" + } + }, + "required": [ + "GeneralInformation" + ] +} diff --git a/pelicun/settings/schema.json b/pelicun/settings/schema.json deleted file mode 100644 index 61b0923f4..000000000 --- a/pelicun/settings/schema.json +++ /dev/null @@ -1,519 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "Options": { - "type": "object", - "properties": { - "Sampling": { - "type": "object", - "properties": { - "SampleSize": { - "type": "integer" - } - } - } - } - }, - "GeneralInformation": { - "type": "object", - "properties": { - "AssetName": { - "type": "string" - }, - "AssetType": { - "type": "string" - }, - "Location": { - "type": "object", - "properties": { - "Latitude": { - "type": "number" - }, - "Longitude": { - "type": "number" - } - }, - "required": [ - "Latitude", - "Longitude" - ] - }, - "units": { - "type": "object", - "properties": { - "length": { - "type": "string", - "examples": [ - "in", - "m" - ] - } - }, - "required": [ - "length" - ] - } - }, - "required": [ - "units" - ] - }, - "assetType": { - "type": "string", - "examples": [ - "Buildings" - ] - }, - "Applications": { - "type": "object", - "properties": { - "DL": { - "type": "object", - "properties": { - "Demands": { - "type": "object", - "properties": { - "DemandFilePath": { - "type": "string" - }, - "SampleSize": { - "type": "integer" - }, - "CoupledDemands": { - "type": "boolean" - }, - "Calibration": { - "type": "object" - }, - "CollapseLimits": { - "type": "object", - "patternProperties": { - ".*": { - "type": "number" - } - } - }, - "InferResidualDrift": { - "type": "object", - "properties": { - "method": { - "type": "string" - }, - "x-direction": { - "type": "number" - }, - "y-direction": { - "type": "number" - } - }, - "required": [ - "method" - ] - } - }, - "required": [ - "DemandFilePath" - ] - }, - "Asset": { - "type": "object", - "properties": { - "ComponentAssignmentFile": { - "type": "string" - }, - "NumberOfStories": { - "type": "integer", - "examples": [ - 1, - 5, - 10 - ] - }, - "ComponentSampleFile": { - "type": "string" - }, - "ComponentDatabase": { - "type": "string" - }, - "ComponentDatabasePath": { - "type": "string" - } - }, - "required": [ - "ComponentAssignmentFile" - ] - }, - "Damage": { - "type": "object", - "properties": { - "CollapseFragility": { - "type": "object", - "properties": { - "DemandType": { - "type": "string" - }, - "CapacityDistribution": { - "type": "string" - }, - "CapacityMedian": { - "type": "number" - }, - "Theta_1": { - "type": "number" - } - }, - "required": [ - "DemandType", - "CapacityDistribution", - "CapacityMedian", - "Theta_1" - ] - }, - "IrreparableDamage": { - "type": "object", - "properties": { - "DriftCapacityMedian": { - "type": "number" - }, - "DriftCapacityLogStd": { - "type": "number" - } - } - } - }, - "required": [ - "CollapseFragility" - ] - }, - "Losses": { - "type": "object", - "properties": { - "Repair": { - "type": "object", - "properties": { - "ConsequenceDatabase": { - "type": "string" - }, - "MapApproach": { - "type": "string" - }, - "MapFilePath": { - "type": "string" - }, - "DecisionVariables": { - "type": "object", - "properties": { - "Cost": { - "type": "boolean" - }, - "Time": { - "type": "boolean" - }, - "Carbon": { - "type": "boolean" - }, - "Energy": { - "type": "boolean" - } - } - }, - "ConsequenceDatabasePath": { - "type": "string" - }, - "ReplacementEnergy": { - "type": "object", - "properties": { - "Unit": { - "type": "string" - }, - "Median": { - "type": "number" - }, - "Distribution": { - "type": "string" - }, - "Theta_1": { - "type": "number" - } - }, - "required": [ - "Unit", - "Median", - "Distribution", - "Theta_1" - ] - }, - "ReplacementCarbon": { - "type": "object", - "properties": { - "Unit": { - "type": "string" - }, - "Median": { - "type": "number" - }, - "Distribution": { - "type": "string" - }, - "Theta_1": { - "type": "number" - } - }, - "required": [ - "Unit", - "Median", - "Distribution", - "Theta_1" - ] - }, - "ReplacementTime": { - "type": "object", - "properties": { - "Unit": { - "type": "string" - }, - "Median": { - "type": "number" - }, - "Distribution": { - "type": "string" - }, - "Theta_1": { - "type": "number" - } - }, - "required": [ - "Unit", - "Median", - "Distribution", - "Theta_1" - ] - }, - "ReplacementCost": { - "type": "object", - "properties": { - "Unit": { - "type": "string" - }, - "Median": { - "type": "number" - }, - "Distribution": { - "type": "string" - }, - "Theta_1": { - "type": "number" - } - }, - "required": [ - "Unit", - "Median", - "Distribution", - "Theta_1" - ] - } - } - } - } - }, - "Outputs": { - "type": "object", - "properties": { - "Demand": { - "type": "object", - "properties": { - "Sample": { - "type": "boolean" - }, - "Statistics": { - "type": "boolean" - } - } - }, - "Asset": { - "type": "object", - "properties": { - "Sample": { - "type": "boolean" - }, - "Statistics": { - "type": "boolean" - } - } - }, - "Damage": { - "type": "object", - "properties": { - "Sample": { - "type": "boolean" - }, - "Statistics": { - "type": "boolean" - }, - "GroupedSample": { - "type": "boolean" - }, - "GroupedStatistics": { - "type": "boolean" - } - } - }, - "Loss": { - "type": "object", - "properties": { - "Repair": { - "type": "object", - "properties": { - "Sample": { - "type": "boolean" - }, - "Statistics": { - "type": "boolean" - }, - "GroupedSample": { - "type": "boolean" - }, - "GroupedStatistics": { - "type": "boolean" - }, - "AggregateSample": { - "type": "boolean" - }, - "AggregateStatistics": { - "type": "boolean" - } - } - } - } - }, - "Format": { - "type": "object", - "properties": { - "CSV": { - "type": "boolean" - }, - "JSON": { - "type": "boolean" - } - } - }, - "Settings": { - "type": "object", - "properties": { - "CondenseDS": { - "type": "boolean" - }, - "SimpleIndexInJSON": { - "type": "boolean" - }, - "AggregateColocatedComponentResults": { - "type": "boolean" - } - } - } - } - }, - "Options": { - "type": "object", - "properties": { - "Options": { - "type": "boolean" - }, - "Seed": { - "type": "integer" - }, - "LogShowMS": { - "type": "boolean" - }, - "LogFile": { - "type": "string" - }, - "UnitsFile": { - "type": "string" - }, - "PrintLog": { - "type": "boolean" - }, - "ShowWarnings": { - "type": "boolean" - }, - "DemandOffset": { - "type": "object" - }, - "ListAllDamageStates": { - "type": "boolean" - }, - "NonDirectionalMultipliers": { - "type": "object" - }, - "EconomiesOfScale": { - "type": "object", - "properties": { - "AcrossFlorrs": { - "type": "boolean" - }, - "AcrossDamageStates": { - "type": "boolean" - } - } - }, - "Sampling": { - "type": "object", - "properties": { - "SamplingMethod": { - "type": "string" - }, - "SampleSize": { - "type": "integer" - }, - "PreserveRawOrder": { - "type": "boolean" - } - } - }, - "RepairCostAndTimeCorrelation": { - "type": "number" - } - } - }, - "DemandAssessment": { - "type": "object", - "properties": { - "Calibration": { - "type": "object", - "properties": { - "Marginals": { - "type": "object" - } - } - } - } - }, - "ApplicationData": { - "type": "object", - "properties": { - "ground_failure": { - "type": "boolean" - } - }, - "required": [ - "ground_failure" - ] - } - } - } - }, - "required": [ - "DL" - ] - }, - "auto_script_path": { - "type": "string" - } - }, - "required": [ - "GeneralInformation", - "Applications" - ] -} diff --git a/pelicun/tests/__init__.py b/pelicun/tests/__init__.py index e69de29bb..1d9bf2ac7 100644 --- a/pelicun/tests/__init__.py +++ b/pelicun/tests/__init__.py @@ -0,0 +1,35 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . + +"""Pelicun Unit Tests.""" diff --git a/pelicun/warnings.py b/pelicun/tests/basic/__init__.py similarity index 85% rename from pelicun/warnings.py rename to pelicun/tests/basic/__init__.py index b7346c465..72c332008 100644 --- a/pelicun/warnings.py +++ b/pelicun/tests/basic/__init__.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California # @@ -33,18 +31,3 @@ # # You should have received a copy of the BSD 3-Clause License along with # pelicun. If not, see . -# -# Contributors: -# Adam Zsarnóczay -# John Vouvakis Manousakis - -""" -This module defines pelicun warning classes and relevant methods - -""" - -from __future__ import annotations - - -class PelicunWarning(Warning): - """Custom warning for specific use in the Pelicun project.""" diff --git a/pelicun/tests/basic/data/base/test_parse_units/duplicate.json b/pelicun/tests/basic/data/base/test_parse_units/duplicate.json index 2fcbf47ca..1baa810f2 100644 --- a/pelicun/tests/basic/data/base/test_parse_units/duplicate.json +++ b/pelicun/tests/basic/data/base/test_parse_units/duplicate.json @@ -39,7 +39,7 @@ "inchps": 0.0254, "ftps": 0.3048 }, - "accelleration": { + "acceleration": { "mps2": 1.0, "inps2": 0.0254, "inchps2": 0.0254, diff --git a/pelicun/tests/basic/data/base/test_parse_units/duplicate2.json b/pelicun/tests/basic/data/base/test_parse_units/duplicate2.json index f0c492e9a..70e60e630 100644 --- a/pelicun/tests/basic/data/base/test_parse_units/duplicate2.json +++ b/pelicun/tests/basic/data/base/test_parse_units/duplicate2.json @@ -38,7 +38,7 @@ "inchps": 0.0254, "ftps": 0.3048 }, - "accelleration": { + "acceleration": { "mps2": 1.0, "inps2": 0.0254, "inchps2": 0.0254, diff --git a/pelicun/tests/basic/data/model/test_DemandModel/_get_required_demand_type/damage_db_testing_utility.csv b/pelicun/tests/basic/data/model/test_DemandModel/_get_required_demand_type/damage_db_testing_utility.csv index 31fa575d7..f3a6811fb 100644 --- a/pelicun/tests/basic/data/model/test_DemandModel/_get_required_demand_type/damage_db_testing_utility.csv +++ b/pelicun/tests/basic/data/model/test_DemandModel/_get_required_demand_type/damage_db_testing_utility.csv @@ -1,2 +1,2 @@ -ID,Incomplete,Demand-Type,Demand-Type_2,Demand-Expression,Demand-Unit,Demand-Offset,Demand-Directional,LS1-Family,LS1-Theta_0,LS1-Theta_1,LS1-DamageStateWeights,LS2-Family,LS2-Theta_0,LS2-Theta_1,LS2-DamageStateWeights,LS3-Family,LS3-Theta_0,LS3-Theta_1,LS3-DamageStateWeights,LS4-Family,LS4-Theta_0,LS4-Theta_1,LS4-DamageStateWeights -testing.component,0,Peak Interstory Drift Ratio,Peak Floor Acceleration,sqrt(X1^2+X2^2),unitless,0,1,lognormal,0.04,0.4,0.950000 | 0.050000,lognormal,0.08,0.4,,lognormal,0.11,0.4,,,,, +ID,Incomplete,Demand-Expression,Demand-Type,Demand-Unit,Demand-Offset,Demand-Directional,Demand2-Type,Demand2-Unit,Demand2-Offset,Demand2-Directional,LS1-Family,LS1-Theta_0,LS1-Theta_1,LS1-DamageStateWeights,LS2-Family,LS2-Theta_0,LS2-Theta_1,LS2-DamageStateWeights,LS3-Family,LS3-Theta_0,LS3-Theta_1,LS3-DamageStateWeights,LS4-Family,LS4-Theta_0,LS4-Theta_1,LS4-DamageStateWeights +testing.component,0,sqrt(X1^2+X2^2),Peak Interstory Drift Ratio,unitless,0,1,Peak Floor Acceleration,unitless,0,1,lognormal,0.04,0.4,0.950000 | 0.050000,lognormal,0.08,0.4,,lognormal,0.11,0.4,,,,, diff --git a/pelicun/tests/basic/reset_tests.py b/pelicun/tests/basic/reset_tests.py index 34fcad11d..8c7db171f 100644 --- a/pelicun/tests/basic/reset_tests.py +++ b/pelicun/tests/basic/reset_tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -38,19 +37,18 @@ # Adam Zsarnóczay # John Vouvakis Manousakis -""" -This file is used to reset all expected test result data. -""" +"""This file is used to reset all expected test result data.""" from __future__ import annotations -import os -import re -import glob + import ast import importlib +import os +import re +from pathlib import Path -def reset_all_test_data(restore=True, purge=False): +def reset_all_test_data(*, restore: bool = True, purge: bool = False) -> None: # noqa: C901 """ Update the expected result pickle files with new results, accepting the values obtained by executing the code as correct from now on. @@ -78,7 +76,6 @@ def reset_all_test_data(restore=True, purge=False): Raises ------ - ValueError If the test directory is not found. @@ -88,18 +85,19 @@ def reset_all_test_data(restore=True, purge=False): `pelicun` directory. Dangerous things may happen otherwise. """ - - cwd = os.path.basename(os.getcwd()) + cwd = Path.cwd() if cwd != 'pelicun': - raise OSError( + msg = ( 'Wrong directory. ' 'See the docstring of `reset_all_test_data`. Aborting' ) + raise OSError(msg) # where the test result data are stored - testdir = os.path.join(*('tests', 'data')) - if not os.path.exists(testdir): - raise ValueError('pelicun/tests/basic/data directory not found.') + testdir = Path('tests') / 'data' + if not testdir.exists(): + msg = 'pelicun/tests/basic/data directory not found.' + raise ValueError(msg) # clean up existing test result data # only remove .pcl files that start with `test_` @@ -108,18 +106,15 @@ def reset_all_test_data(restore=True, purge=False): for root, _, files in os.walk('.'): for filename in files: if pattern.match(filename): - full_name = os.path.join(root, filename) - print(f'removing: {full_name}') - file_path = full_name - os.remove(file_path) + (Path(root) / filename).unlink() # generate new data if restore: # get a list of all existing test files and iterate - test_files = glob.glob('tests/*test*.py') + test_files = list(Path('tests').glob('*test*.py')) for test_file in test_files: # open the file and statically parse the code looking for functions - with open(test_file, 'r', encoding='utf-8') as file: + with Path(test_file).open(encoding='utf-8') as file: node = ast.parse(file.read()) functions = [n for n in node.body if isinstance(n, ast.FunctionDef)] # iterate over the functions looking for test_ functions @@ -131,7 +126,7 @@ def reset_all_test_data(restore=True, purge=False): if 'reset' in arguments: # we want to import it and run it with reset=True # construct a module name, like 'tests.test_uq' - module_name = 'tests.' + os.path.basename(test_file).replace( + module_name = 'tests.' + Path(test_file).name.replace( '.py', '' ) # import the module @@ -139,5 +134,4 @@ def reset_all_test_data(restore=True, purge=False): # get the function func = getattr(module, function.name) # run it to reset its expected test output data - print(f'running: {function.name} from {module_name}') func(reset=True) diff --git a/pelicun/tests/basic/test_assessment.py b/pelicun/tests/basic/test_assessment.py index bec35e975..04e1b7822 100644 --- a/pelicun/tests/basic/test_assessment.py +++ b/pelicun/tests/basic/test_assessment.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -38,27 +37,20 @@ # Adam Zsarnóczay # John Vouvakis Manousakis -""" -These are unit and integration tests on the assessment module of pelicun. -""" +"""These are unit and integration tests on the assessment module of pelicun.""" from __future__ import annotations + import pytest -from pelicun import assessment -# pylint: disable=missing-function-docstring -# pylint: disable=missing-return-doc, missing-return-type-doc +from pelicun import assessment -def create_assessment_obj(config=None): - if config: - asmt = assessment.Assessment(config) - else: - asmt = assessment.Assessment({}) - return asmt +def create_assessment_obj(config: dict | None = None) -> assessment.Assessment: + return assessment.Assessment(config) if config else assessment.Assessment({}) -def test_Assessment_init(): +def test_Assessment_init() -> None: asmt = create_assessment_obj() # confirm attributes for attribute in ( @@ -78,11 +70,10 @@ def test_Assessment_init(): assert hasattr(asmt, attribute) # confirm that creating an attribute on the fly is not allowed with pytest.raises(AttributeError): - # pylint: disable=assigning-non-slot - asmt.my_attribute = 2 + asmt.my_attribute = 2 # type: ignore -def test_assessment_get_default_metadata(): +def test_assessment_get_default_metadata() -> None: asmt = create_assessment_obj() data_sources = ( @@ -101,7 +92,7 @@ def test_assessment_get_default_metadata(): asmt.get_default_metadata(data_source) -def test_assessment_calc_unit_scale_factor(): +def test_assessment_calc_unit_scale_factor() -> None: # default unit file asmt = create_assessment_obj() @@ -135,7 +126,7 @@ def test_assessment_calc_unit_scale_factor(): # 1 smoot was 67 inches in 1958. -def test_assessment_scale_factor(): +def test_assessment_scale_factor() -> None: # default unit file asmt = create_assessment_obj() assert asmt.scale_factor('m') == 1.00 @@ -156,5 +147,5 @@ def test_assessment_scale_factor(): assert asmt.scale_factor('m') == 39.3701 # exceptions - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Unknown unit: helen'): asmt.scale_factor('helen') diff --git a/pelicun/tests/basic/test_asset_model.py b/pelicun/tests/basic/test_asset_model.py index 077087b54..eccde38d0 100644 --- a/pelicun/tests/basic/test_asset_model.py +++ b/pelicun/tests/basic/test_asset_model.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -38,38 +37,39 @@ # Adam Zsarnóczay # John Vouvakis Manousakis -""" -These are unit and integration tests on the asset model of pelicun. -""" +"""These are unit and integration tests on the asset model of pelicun.""" from __future__ import annotations + import tempfile from copy import deepcopy -import pytest +from typing import TYPE_CHECKING + import numpy as np import pandas as pd +import pytest + from pelicun import assessment +from pelicun.base import ensure_value from pelicun.tests.basic.test_pelicun_model import TestPelicunModel -# pylint: disable=missing-function-docstring -# pylint: disable=missing-class-docstring -# pylint: disable=arguments-renamed -# pylint: disable=missing-return-doc,missing-return-type-doc +if TYPE_CHECKING: + from pelicun.model.asset_model import AssetModel class TestAssetModel(TestPelicunModel): @pytest.fixture - def asset_model(self, assessment_instance): + def asset_model(self, assessment_instance: assessment.Assessment) -> AssetModel: return deepcopy(assessment_instance.asset) - def test_init(self, asset_model): + def test_init_method(self, asset_model: AssetModel) -> None: assert asset_model.log assert asset_model.cmp_marginal_params is None assert asset_model.cmp_units is None assert asset_model._cmp_RVs is None assert asset_model.cmp_sample is None - def test_save_cmp_sample(self, asset_model): + def test_save_cmp_sample(self, asset_model: AssetModel) -> None: asset_model.cmp_sample = pd.DataFrame( { ('component_a', f'{i}', f'{j}', '0'): 8.0 @@ -105,10 +105,10 @@ def test_save_cmp_sample(self, asset_model): # also test loading sample to variables # (but we don't inspect them) - _ = asset_model.save_cmp_sample(save_units=False) - _, _ = asset_model.save_cmp_sample(save_units=True) + asset_model.save_cmp_sample(save_units=False) + asset_model.save_cmp_sample(save_units=True) - def test_load_cmp_model_1(self, asset_model): + def test_load_cmp_model_1(self, asset_model: AssetModel) -> None: cmp_marginals = pd.read_csv( 'pelicun/tests/basic/data/model/test_AssetModel/CMP_marginals.csv', index_col=0, @@ -135,7 +135,7 @@ def test_load_cmp_model_1(self, asset_model): pd.testing.assert_frame_equal( expected_cmp_marginal_params, - asset_model.cmp_marginal_params, + ensure_value(asset_model.cmp_marginal_params), check_index_type=False, check_column_type=False, check_dtype=False, @@ -147,11 +147,11 @@ def test_load_cmp_model_1(self, asset_model): pd.testing.assert_series_equal( expected_cmp_units, - asset_model.cmp_units, + ensure_value(asset_model.cmp_units), check_index_type=False, ) - def test_load_cmp_model_2(self, asset_model): + def test_load_cmp_model_2(self, asset_model: AssetModel) -> None: # component marginals utilizing the keywords '--', 'all', 'top', 'roof' cmp_marginals = pd.read_csv( 'pelicun/tests/basic/data/model/test_AssetModel/CMP_marginals_2.csv', @@ -160,7 +160,7 @@ def test_load_cmp_model_2(self, asset_model): asset_model._asmnt.stories = 4 asset_model.load_cmp_model({'marginals': cmp_marginals}) - assert asset_model.cmp_marginal_params.to_dict() == { + assert ensure_value(asset_model.cmp_marginal_params).to_dict() == { 'Theta_0': { ('component_a', '0', '1', '0'): 1.0, ('component_a', '0', '2', '0'): 1.0, @@ -209,23 +209,25 @@ def test_load_cmp_model_2(self, asset_model): pd.testing.assert_series_equal( expected_cmp_units, - asset_model.cmp_units, + ensure_value(asset_model.cmp_units), check_index_type=False, ) - def test_load_cmp_model_csv(self, asset_model): + def test_load_cmp_model_csv(self, asset_model: AssetModel) -> None: # load by directly specifying the csv file cmp_marginals = 'pelicun/tests/basic/data/model/test_AssetModel/CMP' asset_model.load_cmp_model(cmp_marginals) - def test_load_cmp_model_exceptions(self, asset_model): + def test_load_cmp_model_exceptions(self, asset_model: AssetModel) -> None: cmp_marginals = pd.read_csv( 'pelicun/tests/basic/data/model/test_AssetModel/' 'CMP_marginals_invalid_loc.csv', index_col=0, ) asset_model._asmnt.stories = 4 - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match='Cannot parse location string: basement' + ): asset_model.load_cmp_model({'marginals': cmp_marginals}) cmp_marginals = pd.read_csv( @@ -234,10 +236,12 @@ def test_load_cmp_model_exceptions(self, asset_model): index_col=0, ) asset_model._asmnt.stories = 4 - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match='Cannot parse direction string: non-directional' + ): asset_model.load_cmp_model({'marginals': cmp_marginals}) - def test_generate_cmp_sample(self, asset_model): + def test_generate_cmp_sample(self, asset_model: AssetModel) -> None: asset_model.cmp_marginal_params = pd.DataFrame( {'Theta_0': (8.0, 8.0, 8.0, 8.0), 'Blocks': (1.0, 1.0, 1.0, 1.0)}, index=pd.MultiIndex.from_tuples( @@ -278,25 +282,27 @@ def test_generate_cmp_sample(self, asset_model): pd.testing.assert_frame_equal( expected_cmp_sample, - asset_model.cmp_sample, + ensure_value(asset_model.cmp_sample), check_index_type=False, check_column_type=False, ) - def test_generate_cmp_sample_exceptions_1(self, asset_model): + def test_generate_cmp_sample_exceptions_1(self, asset_model: AssetModel) -> None: # without marginal parameters - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match='Model parameters have not been specified' + ): asset_model.generate_cmp_sample(sample_size=10) - def test_generate_cmp_sample_exceptions_2(self, asset_model): + def test_generate_cmp_sample_exceptions_2(self, asset_model: AssetModel) -> None: # without specifying sample size cmp_marginals = pd.read_csv( 'pelicun/tests/basic/data/model/test_AssetModel/CMP_marginals.csv', index_col=0, ) asset_model.load_cmp_model({'marginals': cmp_marginals}) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Sample size was not specified'): asset_model.generate_cmp_sample() # but it should work if a demand sample is available - asset_model._asmnt.demand.sample = np.empty(shape=(10, 2)) + asset_model._asmnt.demand.sample = pd.DataFrame(np.empty(shape=(10, 2))) asset_model.generate_cmp_sample() diff --git a/pelicun/tests/basic/test_auto.py b/pelicun/tests/basic/test_auto.py index 3a96025d0..a91081cc2 100644 --- a/pelicun/tests/basic/test_auto.py +++ b/pelicun/tests/basic/test_auto.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -38,22 +37,16 @@ # Adam Zsarnóczay # John Vouvakis Manousakis -""" -These are unit and integration tests on the auto module of pelicun. - -""" +"""These are unit and integration tests on the auto module of pelicun.""" from __future__ import annotations -from unittest.mock import patch -from unittest.mock import MagicMock -import pytest -from pelicun.auto import auto_populate +from pathlib import Path +from unittest.mock import MagicMock, patch -# pylint: disable=missing-function-docstring -# pylint: disable=missing-return-doc,missing-return-type-doc -# pylint: disable=redefined-outer-name +import pytest +from pelicun.auto import auto_populate # The tests maintain the order of definitions of the `auto.py` file. @@ -67,21 +60,21 @@ @pytest.fixture -def setup_valid_config(): +def setup_valid_config() -> dict: return {'GeneralInformation': {'someKey': 'someValue'}} @pytest.fixture -def setup_auto_script_path(): +def setup_auto_script_path() -> str: return 'PelicunDefault/test_script' @pytest.fixture -def setup_expected_base_path(): +def setup_expected_base_path() -> str: return '/expected/path/resources/auto/' -def test_valid_inputs(setup_valid_config, setup_auto_script_path): +def test_valid_inputs(setup_valid_config: dict, setup_auto_script_path: str) -> None: with patch('pelicun.base.pelicun_path', '/expected/path'), patch( 'os.path.exists', return_value=True ), patch('importlib.__import__') as mock_import: @@ -90,23 +83,23 @@ def test_valid_inputs(setup_valid_config, setup_auto_script_path): ) mock_import.return_value.auto_populate = mock_auto_populate_ext - config, cmp = auto_populate(setup_valid_config, setup_auto_script_path) + config, cmp = auto_populate(setup_valid_config, Path(setup_auto_script_path)) assert 'DL' in config assert cmp == 'CMP' -def test_missing_general_information(): - with pytest.raises(ValueError) as excinfo: - auto_populate({}, 'some/path') - assert "No Asset Information provided for the auto-population routine." in str( - excinfo.value - ) +def test_missing_general_information() -> None: + with pytest.raises( + ValueError, + match='No Asset Information provided for the auto-population routine.', + ): + auto_populate({}, Path('some/path')) def test_pelicun_default_path_replacement( - setup_auto_script_path, setup_expected_base_path -): + setup_auto_script_path: str, setup_expected_base_path: str +) -> None: modified_path = setup_auto_script_path.replace( 'PelicunDefault/', setup_expected_base_path ) @@ -114,8 +107,8 @@ def test_pelicun_default_path_replacement( def test_auto_population_script_execution( - setup_valid_config, setup_auto_script_path -): + setup_valid_config: dict, setup_auto_script_path: str +) -> None: with patch('pelicun.base.pelicun_path', '/expected/path'), patch( 'os.path.exists', return_value=True ), patch('importlib.__import__') as mock_import: @@ -124,5 +117,5 @@ def test_auto_population_script_execution( ) mock_import.return_value.auto_populate = mock_auto_populate_ext - auto_populate(setup_valid_config, setup_auto_script_path) + auto_populate(setup_valid_config, Path(setup_auto_script_path)) mock_import.assert_called_once() diff --git a/pelicun/tests/basic/test_base.py b/pelicun/tests/basic/test_base.py index a5154320e..327ef9c36 100644 --- a/pelicun/tests/basic/test_base.py +++ b/pelicun/tests/basic/test_base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -38,47 +37,49 @@ # Adam Zsarnóczay # John Vouvakis Manousakis -""" -These are unit and integration tests on the base module of pelicun. -""" +"""These are unit and integration tests on the base module of pelicun.""" from __future__ import annotations -import os + +import argparse import io +import platform import re +import subprocess # noqa: S404 import tempfile from contextlib import redirect_stdout -import argparse -import pytest -import pandas as pd +from pathlib import Path + import numpy as np -from pelicun import base +import pandas as pd +import pytest +from pelicun import base +from pelicun.base import ensure_value # The tests maintain the order of definitions of the `base.py` file. -def test_options_init(): - +def test_options_init() -> None: temp_dir = tempfile.mkdtemp() # Create a sample user_config_options dictionary user_config_options = { - "Verbose": False, - "Seed": None, - "LogShowMS": False, - "LogFile": f'{temp_dir}/test_log_file', - "PrintLog": False, - "DemandOffset": {"PFA": -1, "PFV": -1}, - "Sampling": { - "SamplingMethod": "MonteCarlo", - "SampleSize": 1000, - "PreserveRawOrder": False, + 'Verbose': False, + 'Seed': None, + 'LogShowMS': False, + 'LogFile': f'{temp_dir}/test_log_file', + 'PrintLog': False, + 'DemandOffset': {'PFA': -1, 'PFV': -1}, + 'Sampling': { + 'SamplingMethod': 'MonteCarlo', + 'SampleSize': 1000, + 'PreserveRawOrder': False, }, - "SamplingMethod": "MonteCarlo", - "NonDirectionalMultipliers": {"ALL": 1.2}, - "EconomiesOfScale": {"AcrossFloors": True, "AcrossDamageStates": True}, - "RepairCostAndTimeCorrelation": 0.7, + 'SamplingMethod': 'MonteCarlo', + 'NonDirectionalMultipliers': {'ALL': 1.2}, + 'EconomiesOfScale': {'AcrossFloors': True, 'AcrossDamageStates': True}, + 'RepairCostAndTimeCorrelation': 0.7, } # Create an Options object using the user_config_options @@ -95,13 +96,13 @@ def test_options_init(): assert options.demand_offset == {'PFA': -1, 'PFV': -1} assert options.nondir_multi_dict == {'ALL': 1.2} assert options.rho_cost_time == 0.7 - assert options.eco_scale == {"AcrossFloors": True, "AcrossDamageStates": True} + assert options.eco_scale == {'AcrossFloors': True, 'AcrossDamageStates': True} # Check that the Logger object attribute of the Options object is # initialized with the correct parameters assert options.log.verbose is False assert options.log.log_show_ms is False - assert os.path.basename(options.log.log_file) == 'test_log_file' + assert Path(ensure_value(options.log.log_file)).name == 'test_log_file' assert options.log.print_log is False # test seed property and setter @@ -109,16 +110,15 @@ def test_options_init(): assert options.seed == 42 # test rng - # pylint: disable=c-extension-no-member assert isinstance(options.rng, np.random._generator.Generator) -def test_nondir_multi(): +def test_nondir_multi() -> None: options = base.Options({'NonDirectionalMultipliers': {'PFA': 1.5, 'PFV': 1.00}}) assert options.nondir_multi_dict == {'PFA': 1.5, 'PFV': 1.0, 'ALL': 1.2} -def test_logger_init(): +def test_logger_init() -> None: # Test that the Logger object is initialized with the correct # attributes based on the input configuration @@ -130,10 +130,10 @@ def test_logger_init(): 'log_file': f'{temp_dir}/log.txt', 'print_log': True, } - log = base.Logger(**log_config) + log = base.Logger(**log_config) # type: ignore assert log.verbose is True assert log.log_show_ms is False - assert os.path.basename(log.log_file) == 'log.txt' + assert Path(ensure_value(log.log_file)).name == 'log.txt' assert log.print_log is True # test exceptions @@ -144,11 +144,10 @@ def test_logger_init(): 'print_log': True, } with pytest.raises((IsADirectoryError, FileExistsError, FileNotFoundError)): - log = base.Logger(**log_config) + log = base.Logger(**log_config) # type: ignore -def test_logger_msg(): - +def test_logger_msg() -> None: temp_dir = tempfile.mkdtemp() # Test that the msg method prints the correct message to the @@ -159,20 +158,20 @@ def test_logger_msg(): 'log_file': f'{temp_dir}/log.txt', 'print_log': True, } - log = base.Logger(**log_config) + log = base.Logger(**log_config) # type: ignore # Check that the message is printed to the console with io.StringIO() as buf, redirect_stdout(buf): log.msg('This is a message') output = buf.getvalue() assert 'This is a message' in output # Check that the message is written to the log file - with open(f'{temp_dir}/log.txt', 'r', encoding='utf-8') as f: + with Path(f'{temp_dir}/log.txt').open(encoding='utf-8') as f: assert 'This is a message' in f.read() # Check if timestamp is printed with io.StringIO() as buf, redirect_stdout(buf): log.msg( - ('This is a message\nSecond line'), # noqa + ('This is a message\nSecond line'), prepend_timestamp=True, ) output = buf.getvalue() @@ -180,8 +179,7 @@ def test_logger_msg(): assert re.search(pattern, output) is not None -def test_logger_div(): - +def test_logger_div() -> None: temp_dir = tempfile.mkdtemp() # We test the divider with and without the timestamp @@ -199,7 +197,7 @@ def test_logger_div(): 'log_file': f'{temp_dir}/log.txt', 'print_log': True, } - log = base.Logger(**log_config) + log = base.Logger(**log_config) # type: ignore # check console output with io.StringIO() as buf, redirect_stdout(buf): @@ -207,25 +205,87 @@ def test_logger_div(): output = buf.getvalue() assert pattern.match(output) # check log file - with open(f'{temp_dir}/log.txt', 'r', encoding='utf-8') as f: + with Path(f'{temp_dir}/log.txt').open(encoding='utf-8') as f: # simply check that it is not empty assert f.read() -def test_split_file_name(): - file_path = "example.file.name.txt" +@pytest.mark.skipif( + platform.system() == 'Windows', + reason='Skipping test on Windows due to path handling issues.', +) +def test_logger_exception() -> None: + # Create a temporary directory for log files + temp_dir = tempfile.mkdtemp() + + # Create a sample Python script that will raise an exception + test_script = Path(temp_dir) / 'test_script.py' + test_script_content = f""" +from pathlib import Path +from pelicun.base import Logger + +log_file_A = Path("{temp_dir}") / 'log_A.txt' +log_file_B = Path("{temp_dir}") / 'log_B.txt' + +log_A = Logger( + log_file=log_file_A, + verbose=True, + log_show_ms=True, + print_log=True, +) +log_B = Logger( + log_file=log_file_B, + verbose=True, + log_show_ms=True, + print_log=True, +) + +raise ValueError('Test exception in subprocess') +""" + + # Write the test script to the file + test_script.write_text(test_script_content) + + # Use subprocess to run the script + process = subprocess.run( # noqa: S603 + ['python', str(test_script)], # noqa: S607 + capture_output=True, + text=True, + check=False, + ) + + # Check that the process exited with an error + assert process.returncode == 1 + + # Check the stdout/stderr for the expected output + assert 'Test exception in subprocess' in process.stdout + + # Check that the exception was logged in the log file + log_files = ( + Path(temp_dir) / 'log_A.txt', + Path(temp_dir) / 'log_B.txt', + ) + for log_file in log_files: + assert log_file.exists(), 'Log file was not created' + log_content = log_file.read_text() + assert 'Test exception in subprocess' in log_content + assert 'Traceback' in log_content + assert 'ValueError' in log_content + + +def test_split_file_name() -> None: + file_path = 'example.file.name.txt' name, extension = base.split_file_name(file_path) assert name == 'example.file.name' assert extension == '.txt' - file_path = "example" + file_path = 'example' name, extension = base.split_file_name(file_path) assert name == 'example' - assert extension == '' - + assert extension == '' # noqa: PLC1901 -def test_print_system_info(): +def test_print_system_info() -> None: temp_dir = tempfile.mkdtemp() # create a logger object @@ -235,7 +295,7 @@ def test_print_system_info(): 'log_file': f'{temp_dir}/log.txt', 'print_log': True, } - log = base.Logger(**log_config) + log = base.Logger(**log_config) # type: ignore # run print_system_info and get the console output with io.StringIO() as buf, redirect_stdout(buf): @@ -246,7 +306,7 @@ def test_print_system_info(): assert 'System Information:\n' in output -def test_update_vals(): +def test_update_vals() -> None: primary = {'b': {'c': 4, 'd': 5}, 'g': 7} update = {'a': 1, 'b': {'c': 3, 'd': 5}, 'f': 6} base.update_vals(update, primary, 'update', 'primary') @@ -262,18 +322,18 @@ def test_update_vals(): primary = {'a': {'b': 4}} update = {'a': {'b': {'c': 3}}} - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='should not map to a dictionary'): base.update_vals(update, primary, 'update', 'primary') primary = {'a': {'b': 3}} update = {'a': 1, 'b': 2} - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='should map to a dictionary'): base.update_vals(update, primary, 'update', 'primary') -def test_merge_default_config(): +def test_merge_default_config() -> None: # Test merging an empty user config with the default config - user_config = {} + user_config: dict[str, object] | None = {} merged_config = base.merge_default_config(user_config) assert merged_config == base.load_default_options() @@ -302,7 +362,7 @@ def test_merge_default_config(): assert merged_config == {**base.load_default_options(), **user_config} -def test_convert_dtypes(): +def test_convert_dtypes() -> None: # All columns able to be converted # Input DataFrame @@ -368,45 +428,45 @@ def test_convert_dtypes(): ) -def test_convert_to_SimpleIndex(): +def test_convert_to_SimpleIndex() -> None: # Test conversion of a multiindex to a simple index following the # SimCenter dash convention index = pd.MultiIndex.from_tuples((('a', 'b'), ('c', 'd'))) - df = pd.DataFrame([[1, 2], [3, 4]], index=index) - df.index.names = ['name_1', 'name_2'] - df_simple = base.convert_to_SimpleIndex(df, axis=0) - assert df_simple.index.tolist() == ['a-b', 'c-d'] - assert df_simple.index.name == '-'.join(df.index.names) + data = pd.DataFrame([[1, 2], [3, 4]], index=index) + data.index.names = ['name_1', 'name_2'] + data_simple = base.convert_to_SimpleIndex(data, axis=0) + assert data_simple.index.tolist() == ['a-b', 'c-d'] + assert data_simple.index.name == '-'.join(data.index.names) # Test inplace modification - df_inplace = df.copy() + df_inplace = data.copy() base.convert_to_SimpleIndex(df_inplace, axis=0, inplace=True) assert df_inplace.index.tolist() == ['a-b', 'c-d'] - assert df_inplace.index.name == '-'.join(df.index.names) + assert df_inplace.index.name == '-'.join(data.index.names) # Test conversion of columns index = pd.MultiIndex.from_tuples((('a', 'b'), ('c', 'd'))) - df = pd.DataFrame([[1, 2], [3, 4]], columns=index) - df.columns.names = ['name_1', 'name_2'] - df_simple = base.convert_to_SimpleIndex(df, axis=1) - assert df_simple.columns.tolist() == ['a-b', 'c-d'] - assert df_simple.columns.name == '-'.join(df.columns.names) + data = pd.DataFrame([[1, 2], [3, 4]], columns=index) + data.columns.names = ['name_1', 'name_2'] + data_simple = base.convert_to_SimpleIndex(data, axis=1) + assert data_simple.columns.tolist() == ['a-b', 'c-d'] + assert data_simple.columns.name == '-'.join(data.columns.names) # Test inplace modification - df_inplace = df.copy() + df_inplace = data.copy() base.convert_to_SimpleIndex(df_inplace, axis=1, inplace=True) assert df_inplace.columns.tolist() == ['a-b', 'c-d'] - assert df_inplace.columns.name == '-'.join(df.columns.names) + assert df_inplace.columns.name == '-'.join(data.columns.names) # Test invalid axis parameter - with pytest.raises(ValueError): - base.convert_to_SimpleIndex(df, axis=2) + with pytest.raises(ValueError, match='Invalid axis parameter: 2'): + base.convert_to_SimpleIndex(data, axis=2) -def test_convert_to_MultiIndex(): +def test_convert_to_MultiIndex() -> None: # Test a case where the index needs to be converted to a MultiIndex data = pd.DataFrame({'A': (1, 2, 3), 'B': (4, 5, 6)}) - data.index = ('A-1', 'B-1', 'C-1') + data.index = pd.Index(['A-1', 'B-1', 'C-1']) data_converted = base.convert_to_MultiIndex(data, axis=0, inplace=False) expected_index = pd.MultiIndex.from_arrays((('A', 'B', 'C'), ('1', '1', '1'))) assert data_converted.index.equals(expected_index) @@ -414,8 +474,8 @@ def test_convert_to_MultiIndex(): assert data.index.equals(pd.Index(('A-1', 'B-1', 'C-1'))) # Test a case where the index is already a MultiIndex - data_converted = base.convert_to_MultiIndex( - data_converted, axis=0, inplace=False + data_converted = pd.DataFrame( + base.convert_to_MultiIndex(data_converted, axis=0, inplace=False) ) assert data_converted.index.equals(expected_index) @@ -428,42 +488,39 @@ def test_convert_to_MultiIndex(): assert data.columns.equals(pd.Index(('A-1', 'B-1'))) # Test a case where the columns are already a MultiIndex - data_converted = base.convert_to_MultiIndex( - data_converted, axis=1, inplace=False + data_converted = pd.DataFrame( + base.convert_to_MultiIndex(data_converted, axis=1, inplace=False) ) assert data_converted.columns.equals(expected_columns) # Test an invalid axis parameter - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Invalid axis parameter: 2'): base.convert_to_MultiIndex(data_converted, axis=2, inplace=False) # inplace=True data = pd.DataFrame({'A': (1, 2, 3), 'B': (4, 5, 6)}) - data.index = ('A-1', 'B-1', 'C-1') + data.index = pd.Index(['A-1', 'B-1', 'C-1']) base.convert_to_MultiIndex(data, axis=0, inplace=True) expected_index = pd.MultiIndex.from_arrays((('A', 'B', 'C'), ('1', '1', '1'))) assert data.index.equals(expected_index) -def test_show_matrix(): +def test_show_matrix() -> None: # Test with a simple 2D array - arr = ((1, 2, 3), (4, 5, 6)) + arr = np.array(((1, 2, 3), (4, 5, 6))) base.show_matrix(arr) - assert True # if no AssertionError is thrown, then the test passes # Test with a DataFrame - df = pd.DataFrame(((1, 2, 3), (4, 5, 6)), columns=('a', 'b', 'c')) - base.show_matrix(df) - assert True # if no AssertionError is thrown, then the test passes + data = pd.DataFrame(((1, 2, 3), (4, 5, 6)), columns=('a', 'b', 'c')) + base.show_matrix(data) # Test with use_describe=True base.show_matrix(arr, use_describe=True) - assert True # if no AssertionError is thrown, then the test passes -def test_multiply_factor_multiple_levels(): +def test_multiply_factor_multiple_levels() -> None: # Original DataFrame definition - df = pd.DataFrame( + data = pd.DataFrame( np.full((5, 3), 1.00), index=pd.MultiIndex.from_tuples( [ @@ -501,7 +558,7 @@ def test_multiply_factor_multiple_levels(): ), columns=['col1', 'col2', 'col3'], ) - test_df = df.copy() + test_df = data.copy() base.multiply_factor_multiple_levels(test_df, {'lv1': 'A', 'lv2': 'X'}, 2) pd.testing.assert_frame_equal( test_df, @@ -523,7 +580,7 @@ def test_multiply_factor_multiple_levels(): ), columns=['col1', 'col2', 'col3'], ) - test_df = df.copy() + test_df = data.copy() base.multiply_factor_multiple_levels(test_df, {}, 3) pd.testing.assert_frame_equal(test_df, result_df_all) @@ -572,16 +629,14 @@ def test_multiply_factor_multiple_levels(): ) # Test 4: Multiplication with no matching conditions - with pytest.raises(ValueError) as excinfo: - base.multiply_factor_multiple_levels(df.copy(), {'lv1': 'C'}, 2) - assert ( - str(excinfo.value) == "No rows found matching the conditions: `{'lv1': 'C'}`" - ) + with pytest.raises( + ValueError, match="No rows found matching the conditions: `{'lv1': 'C'}`" + ): + base.multiply_factor_multiple_levels(data.copy(), {'lv1': 'C'}, 2) # Test 5: Invalid axis - with pytest.raises(ValueError) as excinfo: - base.multiply_factor_multiple_levels(df.copy(), {'lv1': 'A'}, 2, axis=2) - assert str(excinfo.value) == "Invalid axis: `2`" + with pytest.raises(ValueError, match='Invalid axis: `2`'): + base.multiply_factor_multiple_levels(data.copy(), {'lv1': 'A'}, 2, axis=2) # Test 6: Empty conditions affecting all rows result_df_empty = pd.DataFrame( @@ -598,13 +653,13 @@ def test_multiply_factor_multiple_levels(): ), columns=['col1', 'col2', 'col3'], ) - testing_df = df.copy() + testing_df = data.copy() base.multiply_factor_multiple_levels(testing_df, {}, 4) pd.testing.assert_frame_equal(testing_df, result_df_empty) -def test_describe(): - expected_idx = pd.Index( +def test_describe() -> None: + expected_idx: pd.Index = pd.Index( ( 'count', 'mean', @@ -628,10 +683,10 @@ def test_describe(): # case 1: # passing a DataFrame - df = pd.DataFrame( + data = pd.DataFrame( ((1.00, 2.00, 3.00), (4.00, 5.00, 6.00)), columns=['A', 'B', 'C'] ) - desc = base.describe(df) + desc = base.describe(data) assert np.all(desc.index == expected_idx) assert np.all(desc.columns == pd.Index(('A', 'B', 'C'), dtype='object')) @@ -658,7 +713,7 @@ def test_describe(): assert np.all(desc.columns == pd.Index((0,), dtype='object')) -def test_str2bool(): +def test_str2bool() -> None: assert base.str2bool('True') is True assert base.str2bool('False') is False assert base.str2bool('yes') is True @@ -667,21 +722,21 @@ def test_str2bool(): assert base.str2bool('f') is False assert base.str2bool('1') is True assert base.str2bool('0') is False - assert base.str2bool(True) is True - assert base.str2bool(False) is False + assert base.str2bool(v=True) is True + assert base.str2bool(v=False) is False with pytest.raises(argparse.ArgumentTypeError): base.str2bool('In most cases, it depends..') -def test_float_or_None(): +def test_float_or_None() -> None: # Test with a string that can be converted to a float - assert base.float_or_None('3.14') == 3.14 + assert base.float_or_None('123.00') == 123.00 # Test with a string that represents an integer assert base.float_or_None('42') == 42.0 # Test with a string that represents a negative number - assert base.float_or_None('-3.14') == -3.14 + assert base.float_or_None('-123.00') == -123.00 # Test with a string that can't be converted to a float assert base.float_or_None('hello') is None @@ -690,7 +745,7 @@ def test_float_or_None(): assert base.float_or_None('') is None -def test_int_or_None(): +def test_int_or_None() -> None: # Test the case when the string can be converted to int assert base.int_or_None('123') == 123 assert base.int_or_None('-456') == -456 @@ -704,8 +759,16 @@ def test_int_or_None(): assert base.int_or_None('') is None -def test_with_parsed_str_na_values(): - df = pd.DataFrame( +def test_check_if_str_is_na() -> None: + data = ['N/A', 'foo', 'NaN', '', 'bar', np.nan] + + res = [base.check_if_str_is_na(x) for x in data] + + assert res == [True, False, True, True, False, False] + + +def test_with_parsed_str_na_values() -> None: + data = pd.DataFrame( { 'A': [1.00, 2.00, 'N/A', 4.00, 5.00], 'B': ['foo', 'bar', 'NA', 'baz', 'qux'], @@ -713,7 +776,7 @@ def test_with_parsed_str_na_values(): } ) - res = base.with_parsed_str_na_values(df) + res = base.with_parsed_str_na_values(data) pd.testing.assert_frame_equal( res, pd.DataFrame( @@ -726,17 +789,17 @@ def test_with_parsed_str_na_values(): ) -def test_run_input_specs(): - assert os.path.basename(base.pelicun_path) == 'pelicun' +def test_run_input_specs() -> None: + assert Path(base.pelicun_path).name == 'pelicun' -def test_dedupe_index(): +def test_dedupe_index() -> None: tuples = [('A', '1'), ('A', '1'), ('B', '2'), ('B', '3')] index = pd.MultiIndex.from_tuples(tuples, names=['L1', 'L2']) data = np.full((4, 1), 0.00) - df = pd.DataFrame(data, index=index) - base.dedupe_index(df) - assert df.to_dict() == { + data_pd = pd.DataFrame(data, index=index) + data_pd = base.dedupe_index(data_pd) + assert data_pd.to_dict() == { 0: { ('A', '1', '0'): 0.0, ('A', '1', '1'): 0.0, @@ -746,90 +809,90 @@ def test_dedupe_index(): } -def test_dict_raise_on_duplicates(): +def test_dict_raise_on_duplicates() -> None: res = base.dict_raise_on_duplicates([('A', '1'), ('B', '2')]) assert res == {'A': '1', 'B': '2'} - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='duplicate key: A'): base.dict_raise_on_duplicates([('A', '1'), ('A', '2')]) -def test_parse_units(): +def test_parse_units() -> None: # Test the default units are parsed correctly units = base.parse_units() assert isinstance(units, dict) expect = { - "sec": 1.0, - "minute": 60.0, - "hour": 3600.0, - "day": 86400.0, - "m": 1.0, - "mm": 0.001, - "cm": 0.01, - "km": 1000.0, - "in": 0.0254, - "inch": 0.0254, - "ft": 0.3048, - "mile": 1609.344, - "m2": 1.0, - "mm2": 1e-06, - "cm2": 0.0001, - "km2": 1000000.0, - "in2": 0.00064516, - "inch2": 0.00064516, - "ft2": 0.09290304, - "mile2": 2589988.110336, - "m3": 1.0, - "in3": 1.6387064e-05, - "inch3": 1.6387064e-05, - "ft3": 0.028316846592, - "cmps": 0.01, - "mps": 1.0, - "mph": 0.44704, - "inps": 0.0254, - "inchps": 0.0254, - "ftps": 0.3048, - "mps2": 1.0, - "inps2": 0.0254, - "inchps2": 0.0254, - "ftps2": 0.3048, - "g": 9.80665, - "kg": 1.0, - "ton": 1000.0, - "lb": 0.453592, - "N": 1.0, - "kN": 1000.0, - "lbf": 4.4482179868, - "kip": 4448.2179868, - "kips": 4448.2179868, - "Pa": 1.0, - "kPa": 1000.0, - "MPa": 1000000.0, - "GPa": 1000000000.0, - "psi": 6894.751669043338, - "ksi": 6894751.669043338, - "Mpsi": 6894751669.043338, - "A": 1.0, - "V": 1.0, - "kV": 1000.0, - "ea": 1.0, - "unitless": 1.0, - "rad": 1.0, - "C": 1.0, - "USD_2011": 1.0, - "USD": 1.0, - "loss_ratio": 1.0, - "worker_day": 1.0, - "EA": 1.0, - "SF": 0.09290304, - "LF": 0.3048, - "TN": 1000.0, - "AP": 1.0, - "CF": 0.0004719474432, - "KV": 1000.0, - "J": 1.0, - "MJ": 1000000.0, - "test_two": 2.00, - "test_three": 3.00, + 'sec': 1.0, + 'minute': 60.0, + 'hour': 3600.0, + 'day': 86400.0, + 'm': 1.0, + 'mm': 0.001, + 'cm': 0.01, + 'km': 1000.0, + 'in': 0.0254, + 'inch': 0.0254, + 'ft': 0.3048, + 'mile': 1609.344, + 'm2': 1.0, + 'mm2': 1e-06, + 'cm2': 0.0001, + 'km2': 1000000.0, + 'in2': 0.00064516, + 'inch2': 0.00064516, + 'ft2': 0.09290304, + 'mile2': 2589988.110336, + 'm3': 1.0, + 'in3': 1.6387064e-05, + 'inch3': 1.6387064e-05, + 'ft3': 0.028316846592, + 'cmps': 0.01, + 'mps': 1.0, + 'mph': 0.44704, + 'inps': 0.0254, + 'inchps': 0.0254, + 'ftps': 0.3048, + 'mps2': 1.0, + 'inps2': 0.0254, + 'inchps2': 0.0254, + 'ftps2': 0.3048, + 'g': 9.80665, + 'kg': 1.0, + 'ton': 1000.0, + 'lb': 0.453592, + 'N': 1.0, + 'kN': 1000.0, + 'lbf': 4.4482179868, + 'kip': 4448.2179868, + 'kips': 4448.2179868, + 'Pa': 1.0, + 'kPa': 1000.0, + 'MPa': 1000000.0, + 'GPa': 1000000000.0, + 'psi': 6894.751669043338, + 'ksi': 6894751.669043338, + 'Mpsi': 6894751669.043338, + 'A': 1.0, + 'V': 1.0, + 'kV': 1000.0, + 'ea': 1.0, + 'unitless': 1.0, + 'rad': 1.0, + 'C': 1.0, + 'USD_2011': 1.0, + 'USD': 1.0, + 'loss_ratio': 1.0, + 'worker_day': 1.0, + 'EA': 1.0, + 'SF': 0.09290304, + 'LF': 0.3048, + 'TN': 1000.0, + 'AP': 1.0, + 'CF': 0.0004719474432, + 'KV': 1000.0, + 'J': 1.0, + 'MJ': 1000000.0, + 'test_two': 2.00, + 'test_three': 3.00, } for thing, value in units.items(): assert thing in expect @@ -851,7 +914,10 @@ def test_parse_units(): # Test that an exception is raised if the additional units file is # not a valid JSON file invalid_json_file = 'pelicun/tests/basic/data/base/test_parse_units/invalid.json' - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match='not a valid JSON file.', + ): units = base.parse_units(invalid_json_file) # Test that an exception is raised if a unit is defined twice in @@ -859,7 +925,10 @@ def test_parse_units(): duplicate_units_file = ( 'pelicun/tests/basic/data/base/test_parse_units/duplicate2.json' ) - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match='sec defined twice', + ): units = base.parse_units(duplicate_units_file) # Test that an exception is raised if a unit conversion factor is not a float @@ -874,11 +943,14 @@ def test_parse_units(): invalid_units_file = ( 'pelicun/tests/basic/data/base/test_parse_units/not_dict.json' ) - with pytest.raises(ValueError): + with pytest.raises( + (ValueError, TypeError), + match="contains first-level keys that don't point to a dictionary", + ): units = base.parse_units(invalid_units_file) -def test_unit_conversion(): +def test_unit_conversion() -> None: # Test scalar conversion from feet to meters assert base.convert_units(1.00, 'ft', 'm') == 0.3048 @@ -901,46 +973,44 @@ def test_unit_conversion(): # Test error handling for invalid input type with pytest.raises(TypeError) as excinfo: - base.convert_units("one", 'ft', 'm') + base.convert_units('one', 'ft', 'm') # type: ignore assert str(excinfo.value) == 'Invalid input type for `values`' # Test error handling for unknown unit - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match='Unknown unit `xyz`'): base.convert_units(1.00, 'xyz', 'm') - assert str(excinfo.value) == 'Unknown unit `xyz`' # Test error handling for mismatched category - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match='Unknown unit: `ft`'): base.convert_units(1.00, 'ft', 'm', category='volume') - assert str(excinfo.value) == 'Unknown unit: `ft`' # Test error handling unknown category - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match='Unknown category: `unknown_category`'): base.convert_units(1.00, 'ft', 'm', category='unknown_category') - assert str(excinfo.value) == 'Unknown category: `unknown_category`' # Test error handling different categories - with pytest.raises(ValueError) as excinfo: + with pytest.raises( + ValueError, + match='`lb` is a `mass` unit, but `m` is not specified in that category.', + ): base.convert_units(1.00, 'lb', 'm') - assert ( - str(excinfo.value) - == '`lb` is a `mass` unit, but `m` is not specified in that category.' - ) -def test_stringterpolation(): +def test_stringterpolation() -> None: func = base.stringterpolation('1,2,3|4,5,6') x_new = np.array([4, 4.5, 5]) expected = np.array([1, 1.5, 2]) np.testing.assert_array_almost_equal(func(x_new), expected) -def test_invert_mapping(): +def test_invert_mapping() -> None: original_dict = {'a': [1, 2], 'b': [3]} expected = {1: 'a', 2: 'a', 3: 'b'} assert base.invert_mapping(original_dict) == expected # with duplicates, raises an error original_dict = {'a': [1, 2], 'b': [2]} - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match='Cannot invert mapping with duplicate values.' + ): base.invert_mapping(original_dict) diff --git a/pelicun/tests/basic/test_damage_model.py b/pelicun/tests/basic/test_damage_model.py index 1e991408f..51d576763 100644 --- a/pelicun/tests/basic/test_damage_model.py +++ b/pelicun/tests/basic/test_damage_model.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -38,55 +37,57 @@ # Adam Zsarnóczay # John Vouvakis Manousakis -""" -These are unit and integration tests on the damage model of pelicun. -""" +"""These are unit and integration tests on the damage model of pelicun.""" from __future__ import annotations -from copy import deepcopy + import warnings -import pytest +from copy import deepcopy +from typing import TYPE_CHECKING + import numpy as np import pandas as pd -from pelicun import base -from pelicun import uq -from pelicun.model.damage_model import DamageModel -from pelicun.model.damage_model import DamageModel_Base -from pelicun.model.damage_model import DamageModel_DS -from pelicun.model.damage_model import _is_for_ds_model +import pytest +from scipy.stats import norm + +from pelicun import base, uq +from pelicun.base import ensure_value +from pelicun.model.damage_model import ( + DamageModel, + DamageModel_Base, + DamageModel_DS, + _is_for_ds_model, +) +from pelicun.pelicun_warnings import PelicunWarning from pelicun.tests.basic.test_pelicun_model import TestPelicunModel -from pelicun.warnings import PelicunWarning -# pylint: disable=missing-function-docstring -# pylint: disable=missing-class-docstring -# pylint: disable=missing-return-doc,missing-return-type-doc +if TYPE_CHECKING: + from pelicun.assessment import Assessment class TestDamageModel(TestPelicunModel): - @pytest.fixture - def damage_model(self, assessment_instance): + def damage_model(self, assessment_instance: Assessment) -> DamageModel: return deepcopy(assessment_instance.damage) - def test___init__(self, damage_model): + def test___init__(self, damage_model: DamageModel) -> None: assert damage_model.log assert damage_model.ds_model with pytest.raises(AttributeError): - damage_model.xyz = 123 + damage_model.xyz = 123 # type: ignore assert damage_model.ds_model.damage_params is None assert damage_model.ds_model.sample is None assert len(damage_model._damage_models) == 1 - def test_damage_models(self, assessment_instance): - + def test_damage_models(self, assessment_instance: Assessment) -> None: damage_model = DamageModel(assessment_instance) assert damage_model._damage_models is not None assert len(damage_model._damage_models) == 1 assert isinstance(damage_model._damage_models[0], DamageModel_DS) - def test_load_model_parameters(self, damage_model): + def test_load_model_parameters(self, damage_model: DamageModel) -> None: path = ( 'pelicun/tests/basic/data/model/test_DamageModel/' 'load_model_parameters/damage_db.csv' @@ -100,11 +101,12 @@ def test_load_model_parameters(self, damage_model): damage_model.load_model_parameters([path], cmp_set, warn_missing=True) assert len(w) == 1 assert ( - "The damage model does not provide damage information " - "for the following component(s) in the asset model: " + 'The damage model does not provide damage information ' + 'for the following component(s) in the asset model: ' "['component.incomplete']." ) in str(w[0].message) damage_parameters = damage_model.ds_model.damage_params + assert damage_parameters is not None assert 'component.A' in damage_parameters.index assert 'component.B' in damage_parameters.index assert 'component.C' not in damage_parameters.index @@ -127,29 +129,28 @@ def test_load_model_parameters(self, damage_model): damage_model.load_model_parameters([path], cmp_set, warn_missing=True) assert len(w) == 1 assert ( - "The damage model does not provide damage " - "information for the following component(s) " + 'The damage model does not provide damage ' + 'information for the following component(s) ' "in the asset model: ['not.exist']." ) in str(w[0].message) - assert damage_model.ds_model.damage_params.empty + assert ensure_value(damage_model.ds_model.damage_params).empty - def test_calculate(self): + def test_calculate(self) -> None: # User-facing methods are coupled with other assessment objects # and are tested in the verification examples. pass - def test_save_sample(self): + def test_save_sample(self) -> None: # User-facing methods are coupled with other assessment objects # and are tested in the verification examples. pass - def test_load_sample(self): + def test_load_sample(self) -> None: # User-facing methods are coupled with other assessment objects # and are tested in the verification examples. pass - def test__get_component_id_set(self, assessment_instance): - + def test__get_component_id_set(self, assessment_instance: Assessment) -> None: damage_model = DamageModel(assessment_instance) damage_model.ds_model.damage_params = pd.DataFrame( @@ -166,8 +167,9 @@ def test__get_component_id_set(self, assessment_instance): assert component_id_set == expected_set - def test__ensure_damage_parameter_availability(self, assessment_instance): - + def test__ensure_damage_parameter_availability( + self, assessment_instance: Assessment + ) -> None: damage_model = DamageModel(assessment_instance) damage_model.ds_model.damage_params = pd.DataFrame( @@ -178,29 +180,26 @@ def test__ensure_damage_parameter_availability(self, assessment_instance): index=pd.Index(['cmp.1', 'cmp.2', 'cmp.3'], name='ID'), ) - cmp_list = ['cmp.1', 'cmp.2', 'cmp.3', 'cmp.4'] + cmp_set = {'cmp.1', 'cmp.2', 'cmp.3', 'cmp.4'} expected_missing_components = ['cmp.4'] with pytest.warns(PelicunWarning) as record: missing_components = damage_model._ensure_damage_parameter_availability( - cmp_list, warn_missing=True + cmp_set, warn_missing=True ) assert missing_components == expected_missing_components assert len(record) == 1 - assert "cmp.4" in str(record[0].message) + assert 'cmp.4' in str(record[0].message) class TestDamageModel_Base(TestPelicunModel): - def test___init__(self, assessment_instance): - + def test___init__(self, assessment_instance: Assessment) -> None: damage_model = DamageModel_Base(assessment_instance) with pytest.raises(AttributeError): - # pylint: disable=assigning-non-slot - damage_model.xyz = 123 - - def test__load_model_parameters(self, assessment_instance): + damage_model.xyz = 123 # type: ignore + def test__load_model_parameters(self, assessment_instance: Assessment) -> None: damage_model = DamageModel_Base(assessment_instance) damage_model.damage_params = pd.DataFrame( @@ -221,7 +220,7 @@ def test__load_model_parameters(self, assessment_instance): index=pd.Index(['cmp.1', 'cmp.3'], name='ID'), ) - damage_model._load_model_parameters(new_data) + damage_model.load_model_parameters(new_data) pd.testing.assert_frame_equal( damage_model.damage_params, @@ -234,12 +233,13 @@ def test__load_model_parameters(self, assessment_instance): ), ) - def test__convert_damage_parameter_units(self, assessment_instance): - + def test__convert_damage_parameter_units( + self, assessment_instance: Assessment + ) -> None: damage_model = DamageModel_Base(assessment_instance) # should have no effect when damage_params is None - damage_model._convert_damage_parameter_units() + damage_model.convert_damage_parameter_units() # converting units from 'g' to 'm/s2' (1g ~ 9.80665 m/s2) @@ -251,7 +251,7 @@ def test__convert_damage_parameter_units(self, assessment_instance): index=pd.Index(['cmp.1', 'cmp.2'], name='ID'), ) - damage_model._convert_damage_parameter_units() + damage_model.convert_damage_parameter_units() pd.testing.assert_frame_equal( damage_model.damage_params, @@ -266,13 +266,14 @@ def test__convert_damage_parameter_units(self, assessment_instance): ), ) - def test__remove_incomplete_components(self, assessment_instance): - + def test__remove_incomplete_components( + self, assessment_instance: Assessment + ) -> None: damage_model = DamageModel_Base(assessment_instance) # with damage_model.damage_params set to None this should have # no effect. - damage_model._remove_incomplete_components() + damage_model.remove_incomplete_components() damage_model.damage_params = pd.DataFrame( { @@ -282,7 +283,7 @@ def test__remove_incomplete_components(self, assessment_instance): index=pd.Index(['cmp.1', 'cmp.2', 'cmp.3', 'cmp.4'], name='ID'), ) - damage_model._remove_incomplete_components() + damage_model.remove_incomplete_components() pd.testing.assert_frame_equal( damage_model.damage_params, @@ -298,14 +299,17 @@ def test__remove_incomplete_components(self, assessment_instance): # with damage_model.damage_params set to None this should have # no effect. - damage_model.damage_params.drop(('Incomplete', ''), axis=1, inplace=True) + damage_model.damage_params = damage_model.damage_params.drop( + ('Incomplete', ''), axis=1 + ) # now, this should also have no effect before = damage_model.damage_params.copy() - damage_model._remove_incomplete_components() + damage_model.remove_incomplete_components() pd.testing.assert_frame_equal(before, damage_model.damage_params) - def test__drop_unused_damage_parameters(self, assessment_instance): - + def test__drop_unused_damage_parameters( + self, assessment_instance: Assessment + ) -> None: damage_model = DamageModel_Base(assessment_instance) damage_model.damage_params = pd.DataFrame( @@ -314,15 +318,14 @@ def test__drop_unused_damage_parameters(self, assessment_instance): cmp_set = {'cmp.1', 'cmp.3'} - damage_model._drop_unused_damage_parameters(cmp_set) + damage_model.drop_unused_damage_parameters(cmp_set) pd.testing.assert_frame_equal( damage_model.damage_params, pd.DataFrame(index=pd.Index(['cmp.1', 'cmp.3'], name='ID')), ) - def test__get_pg_batches(self, assessment_instance): - + def test__get_pg_batches(self, assessment_instance: Assessment) -> None: damage_model = DamageModel_Base(assessment_instance) component_blocks = pd.DataFrame( @@ -370,9 +373,7 @@ def test__get_pg_batches(self, assessment_instance): class TestDamageModel_DS(TestDamageModel_Base): - - def test__obtain_ds_sample(self, assessment_instance): - + def test__obtain_ds_sample(self, assessment_instance: Assessment) -> None: damage_model = DamageModel_DS(assessment_instance) demand_sample = pd.DataFrame( @@ -417,7 +418,7 @@ def test__obtain_ds_sample(self, assessment_instance): index=['cmp.1', 'cmp.2', 'cmp.3'], ).rename_axis('ID') - damage_model._obtain_ds_sample( + damage_model.obtain_ds_sample( demand_sample, component_blocks, block_batch_size, @@ -426,7 +427,7 @@ def test__obtain_ds_sample(self, assessment_instance): nondirectional_multipliers, ) pd.testing.assert_frame_equal( - damage_model.ds_sample, + ensure_value(damage_model.ds_sample), pd.DataFrame( { ('cmp.1', '1', '1', '1', '1'): [1, 1], @@ -438,8 +439,7 @@ def test__obtain_ds_sample(self, assessment_instance): ).rename_axis(columns=['cmp', 'loc', 'dir', 'uid', 'block']), ) - def test__handle_operation(self, assessment_instance): - + def test__handle_operation(self, assessment_instance: Assessment) -> None: damage_model = DamageModel_DS(assessment_instance) assert damage_model._handle_operation(1.00, '+', 1.00) == 2.00 @@ -447,16 +447,14 @@ def test__handle_operation(self, assessment_instance): assert damage_model._handle_operation(1.00, '*', 4.00) == 4.00 assert damage_model._handle_operation(8.00, '/', 8.00) == 1.00 - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match='Invalid operation: `%`'): damage_model._handle_operation(1.00, '%', 1.00) - assert ('Invalid operation: `%`') in str(record.value) - - def test__generate_dmg_sample(self, assessment_instance): + def test__generate_dmg_sample(self, assessment_instance: Assessment) -> None: # Create an instance of the damage model damage_model = DamageModel_DS(assessment_instance) - PGB = pd.DataFrame( + pgb = pd.DataFrame( {'Blocks': [1]}, index=pd.MultiIndex.from_tuples( [('cmp.test', '1', '2', '3')], @@ -484,7 +482,7 @@ def test__generate_dmg_sample(self, assessment_instance): sample_size = 2 capacity_sample, lsds_sample = damage_model._generate_dmg_sample( - sample_size, PGB, scaling_specification + sample_size, pgb, scaling_specification ) pd.testing.assert_frame_equal( @@ -506,15 +504,15 @@ def test__generate_dmg_sample(self, assessment_instance): ).rename_axis(columns=['cmp', 'loc', 'dir', 'uid', 'block', 'ls']), ) - def test__create_dmg_RVs(self, assessment_instance): - + def test__create_dmg_RVs(self, assessment_instance: Assessment) -> None: damage_model = DamageModel_DS(assessment_instance) - PGB = pd.DataFrame( + pgb = pd.DataFrame( {'Blocks': [1]}, index=pd.MultiIndex.from_tuples( [ ('cmp.A', '1', '2', '3'), + ('cmp.B', '1', '2', '3'), ], names=['cmp', 'loc', 'dir', 'uid'], ), @@ -522,29 +520,37 @@ def test__create_dmg_RVs(self, assessment_instance): damage_params = pd.DataFrame( { - ('Demand', 'Directional'): [0.0], - ('Demand', 'Offset'): [0.0], - ('Demand', 'Type'): ['Peak Floor Acceleration'], - ('Incomplete', ''): [0], + ('Demand', 'Directional'): [0.0, 0.0], + ('Demand', 'Offset'): [0.0, 0.0], + ('Demand', 'Type'): [ + 'Peak Floor Acceleration', + 'Peak Floor Acceleration', + ], + ('Incomplete', ''): [0, 0], ('LS1', 'DamageStateWeights'): [ '0.40 | 0.10 | 0.50', + '0.40 | 0.10 | 0.50', ], - ('LS1', 'Family'): ['lognormal'], - ('LS1', 'Theta_0'): [30.00], - ('LS1', 'Theta_1'): [0.5], + ('LS1', 'Family'): ['lognormal', 'lognormal'], + ('LS1', 'Theta_0'): [30.00, 30.00], + ('LS1', 'Theta_1'): [0.5, 0.5], }, - index=['cmp.A'], + index=['cmp.A', 'cmp.B'], ).rename_axis('ID') # Attach this DataFrame to the damage model instance damage_model.damage_params = damage_params # Define a scaling specification - scaling_specification = {'cmp.A-1-2': '*1.20'} - - # Execute the method under test - capacity_RV_reg, lsds_RV_reg = damage_model._create_dmg_RVs( - PGB, scaling_specification + operation_list = ['*1.20', '+0.10', '/1.20', '-0.10', '*1.10'] + scaling_specification = { + 'cmp.A-1-2': {'LS1': '*1.20'}, + 'cmp.B-1-2': {'LS1': operation_list}, + } + + # Create random variables based on the damage parameters + capacity_rv_reg, lsds_rv_reg = damage_model._create_dmg_RVs( + pgb, scaling_specification ) # Now we need to verify the outputs in the registries @@ -552,20 +558,51 @@ def test__create_dmg_RVs(self, assessment_instance): # created correctly. # Example check for presence and properties of a # RandomVariable in the registry: - assert 'FRG-cmp.A-1-2-3-1-1' in capacity_RV_reg.RV + assert 'FRG-cmp.A-1-2-3-1-1' in capacity_rv_reg.RV + assert 'FRG-cmp.B-1-2-3-1-1' in capacity_rv_reg.RV assert isinstance( - capacity_RV_reg.RV['FRG-cmp.A-1-2-3-1-1'], + capacity_rv_reg.RV['FRG-cmp.A-1-2-3-1-1'], + uq.LogNormalRandomVariable, + ) + assert isinstance( + capacity_rv_reg.RV['FRG-cmp.B-1-2-3-1-1'], uq.LogNormalRandomVariable, ) - assert 'LSDS-cmp.A-1-2-3-1-1' in lsds_RV_reg.RV + assert 'LSDS-cmp.A-1-2-3-1-1' in lsds_rv_reg.RV + assert 'LSDS-cmp.B-1-2-3-1-1' in lsds_rv_reg.RV assert isinstance( - lsds_RV_reg.RV['LSDS-cmp.A-1-2-3-1-1'], + lsds_rv_reg.RV['LSDS-cmp.A-1-2-3-1-1'], + uq.MultinomialRandomVariable, + ) + assert isinstance( + lsds_rv_reg.RV['LSDS-cmp.B-1-2-3-1-1'], uq.MultinomialRandomVariable, ) - def test__evaluate_damage_state(self, assessment_instance): - + # Validate the scaling of the random variables are correct + # Generate samples for validating that theta_0 is scaled correctly + capacity_rv_reg.generate_sample( + sample_size=len(operation_list), method='LHS' + ) + cmp_b_scaled_theta0 = np.array( + [30.0 * 1.20, 30.0 + 0.10, 30.0 / 1.20, 30.0 - 0.10, 30.0 * 1.10] + ) + for rv_name, rv in capacity_rv_reg.RV.items(): + uniform_sample = rv._uni_sample + sample = rv.sample + assert uniform_sample is not None + assert sample is not None + for i in range(len(operation_list)): + if rv_name == 'FRG-cmp.A-1-2-3-1-1': + theta = 1.20 * 30.0 + elif rv_name == 'FRG-cmp.B-1-2-3-1-1': + theta = cmp_b_scaled_theta0[i] + assert sample[i] == np.exp( + norm.ppf(uniform_sample[i], loc=np.log(theta), scale=0.5) + ) + + def test__evaluate_damage_state(self, assessment_instance: Assessment) -> None: # We define a single component with 3 limit states. # The last limit state can have two damage states, DS3 and DS4. # We test that the damage state assignments are correct. @@ -609,8 +646,7 @@ def test__evaluate_damage_state(self, assessment_instance): ).rename_axis(columns=['cmp', 'loc', 'dir', 'uid', 'block']), ) - def test__prepare_dmg_quantities(self, assessment_instance): - + def test__prepare_dmg_quantities(self, assessment_instance: Assessment) -> None: # # A case with blocks # @@ -638,10 +674,10 @@ def test__prepare_dmg_quantities(self, assessment_instance): index=pd.MultiIndex.from_tuples([('A', '0', '1', '0')]), ).rename_axis(index=['cmp', 'loc', 'dir', 'uid']) - res = damage_model._prepare_dmg_quantities( + res = damage_model.prepare_dmg_quantities( component_sample, component_marginal_parameters, - True, + dropzero=True, ) # Each block takes half the quantity. @@ -678,10 +714,10 @@ def test__prepare_dmg_quantities(self, assessment_instance): }, ).rename_axis(columns=['cmp', 'loc', 'dir', 'uid']) - res = damage_model._prepare_dmg_quantities( + res = damage_model.prepare_dmg_quantities( component_sample, None, - True, + dropzero=True, ) # Realization 0: Expect NaNs @@ -719,10 +755,10 @@ def test__prepare_dmg_quantities(self, assessment_instance): }, ).rename_axis(columns=['cmp', 'loc', 'dir', 'uid']) - res = damage_model._prepare_dmg_quantities( + res = damage_model.prepare_dmg_quantities( component_sample, None, - True, + dropzero=True, ) pd.testing.assert_frame_equal( @@ -734,10 +770,10 @@ def test__prepare_dmg_quantities(self, assessment_instance): ).rename_axis(columns=['cmp', 'loc', 'dir', 'uid', 'ds']), ) - res = damage_model._prepare_dmg_quantities( + res = damage_model.prepare_dmg_quantities( component_sample, None, - False, + dropzero=False, ) pd.testing.assert_frame_equal( @@ -751,8 +787,7 @@ def test__prepare_dmg_quantities(self, assessment_instance): ).rename_axis(columns=['cmp', 'loc', 'dir', 'uid', 'ds']), ) - def test__perform_dmg_task(self, assessment_instance): - + def test__perform_dmg_task(self, assessment_instance: Assessment) -> None: # noqa: C901 damage_model = DamageModel_DS(assessment_instance) # @@ -769,9 +804,9 @@ def test__perform_dmg_task(self, assessment_instance): dtype='int32', ).rename_axis(columns=['cmp', 'loc', 'dir', 'uid']) - dmg_process = {"1_CMP.B": {"DS1": "CMP.A_DS4"}} + dmg_process = {'1_CMP.B': {'DS1': 'CMP.A_DS4'}} for task in dmg_process.items(): - damage_model._perform_dmg_task(task) + damage_model.perform_dmg_task(task) pd.testing.assert_frame_equal( damage_model.ds_sample, @@ -800,9 +835,9 @@ def test__perform_dmg_task(self, assessment_instance): dtype='int32', ).rename_axis(columns=['cmp', 'loc', 'dir', 'uid']) - dmg_process = {"1_CMP.B": {"DS1": "CMP.A_NA"}} + dmg_process = {'1_CMP.B': {'DS1': 'CMP.A_NA'}} for task in dmg_process.items(): - damage_model._perform_dmg_task(task) + damage_model.perform_dmg_task(task) pd.testing.assert_frame_equal( damage_model.ds_sample, @@ -833,9 +868,9 @@ def test__perform_dmg_task(self, assessment_instance): dtype='int32', ).rename_axis(columns=['cmp', 'loc', 'dir', 'uid']) - dmg_process = {"1_CMP.B-LOC": {"DS1": "CMP.A_DS4"}} + dmg_process = {'1_CMP.B-LOC': {'DS1': 'CMP.A_DS4'}} for task in dmg_process.items(): - damage_model._perform_dmg_task(task) + damage_model.perform_dmg_task(task) pd.testing.assert_frame_equal( damage_model.ds_sample, @@ -867,9 +902,9 @@ def test__perform_dmg_task(self, assessment_instance): dtype='int32', ).rename_axis(columns=['cmp', 'loc', 'dir', 'uid']) - dmg_process = {"1_CMP.A": {"DS1": "ALL_DS2"}} + dmg_process = {'1_CMP.A': {'DS1': 'ALL_DS2'}} for task in dmg_process.items(): - damage_model._perform_dmg_task(task) + damage_model.perform_dmg_task(task) pd.testing.assert_frame_equal( damage_model.ds_sample, @@ -900,9 +935,9 @@ def test__perform_dmg_task(self, assessment_instance): dtype='int32', ).rename_axis(columns=['cmp', 'loc', 'dir', 'uid']) - dmg_process = {"1_CMP.B": {"DS1": "CMP.A_NA"}} + dmg_process = {'1_CMP.B': {'DS1': 'CMP.A_NA'}} for task in dmg_process.items(): - damage_model._perform_dmg_task(task) + damage_model.perform_dmg_task(task) pd.testing.assert_frame_equal( damage_model.ds_sample, @@ -931,9 +966,9 @@ def test__perform_dmg_task(self, assessment_instance): dtype='int32', ).rename_axis(columns=['cmp', 'loc', 'dir', 'uid']) - dmg_process = {"1_CMP.B-LOC": {"DS1": "CMP.A_NA"}} + dmg_process = {'1_CMP.B-LOC': {'DS1': 'CMP.A_NA'}} for task in dmg_process.items(): - damage_model._perform_dmg_task(task) + damage_model.perform_dmg_task(task) pd.testing.assert_frame_equal( damage_model.ds_sample, @@ -964,9 +999,9 @@ def test__perform_dmg_task(self, assessment_instance): dtype='int32', ).rename_axis(columns=['cmp', 'loc', 'dir', 'uid']) - dmg_process = {"1_CMP.A-LOC": {"DS1": "ALL_NA"}} + dmg_process = {'1_CMP.A-LOC': {'DS1': 'ALL_NA'}} for task in dmg_process.items(): - damage_model._perform_dmg_task(task) + damage_model.perform_dmg_task(task) pd.testing.assert_frame_equal( damage_model.ds_sample, @@ -994,10 +1029,10 @@ def test__perform_dmg_task(self, assessment_instance): dtype='int32', ).rename_axis(columns=['cmp', 'loc', 'dir', 'uid']) - dmg_process = {"1_CMP.C": {"DS1": "CMP.A_DS4"}} + dmg_process = {'1_CMP.C': {'DS1': 'CMP.A_DS4'}} with pytest.warns(PelicunWarning) as record: for task in dmg_process.items(): - damage_model._perform_dmg_task(task) + damage_model.perform_dmg_task(task) assert ( 'Source component `CMP.C` in the prescribed damage process not found' ) in str(record.list[0].message) @@ -1005,10 +1040,10 @@ def test__perform_dmg_task(self, assessment_instance): # # Test warnings: Target component not found # - dmg_process = {"1_CMP.A": {"DS1": "CMP.C_DS4"}} + dmg_process = {'1_CMP.A': {'DS1': 'CMP.C_DS4'}} with pytest.warns(PelicunWarning) as record: for task in dmg_process.items(): - damage_model._perform_dmg_task(task) + damage_model.perform_dmg_task(task) assert ( 'Target component `CMP.C` in the prescribed damage process not found' ) in str(record.list[0].message) @@ -1016,23 +1051,22 @@ def test__perform_dmg_task(self, assessment_instance): # # Test Error: Unable to parse source event # - dmg_process = {"1_CMP.A": {"XYZ": "CMP.B_DS1"}} - with pytest.raises(ValueError) as record: - for task in dmg_process.items(): - damage_model._perform_dmg_task(task) - assert ('Unable to parse source event in damage process: `XYZ`') in str( - record.value - ) - dmg_process = {"1_CMP.A": {"DS1": "CMP.B_ABC"}} - with pytest.raises(ValueError) as record: - for task in dmg_process.items(): - damage_model._perform_dmg_task(task) - assert ('Unable to parse target event in damage process: `ABC`') in str( - record.value - ) - - def test__complete_ds_cols(self, assessment_instance): + dmg_process = {'1_CMP.A': {'XYZ': 'CMP.B_DS1'}} + for task in dmg_process.items(): + with pytest.raises( + ValueError, + match='Unable to parse source event in damage process: `XYZ`', + ): + damage_model.perform_dmg_task(task) + dmg_process = {'1_CMP.A': {'DS1': 'CMP.B_ABC'}} + for task in dmg_process.items(): + with pytest.raises( + ValueError, + match='Unable to parse target event in damage process: `ABC`', + ): + damage_model.perform_dmg_task(task) + def test__complete_ds_cols(self, assessment_instance: Assessment) -> None: damage_model = DamageModel_DS(assessment_instance) # the method needs damage parameters damage_model.damage_params = base.convert_to_MultiIndex( @@ -1053,7 +1087,7 @@ def test__complete_ds_cols(self, assessment_instance): ('single.ds', '0', '0', '0', '1'): [100.00], }, ).rename_axis(columns=['cmp', 'loc', 'dir', 'uid', 'ds']) - out = damage_model._complete_ds_cols(dmg_sample) + out = damage_model.complete_ds_cols(dmg_sample) pd.testing.assert_frame_equal( out, pd.DataFrame( @@ -1069,8 +1103,7 @@ def test__complete_ds_cols(self, assessment_instance): ) -def test__is_for_ds_model(): - +def test__is_for_ds_model() -> None: data_with_ls1 = pd.DataFrame( { ('LS1', 'Theta_0'): [0.5], diff --git a/pelicun/tests/basic/test_demand_model.py b/pelicun/tests/basic/test_demand_model.py index 217512c4a..020084313 100644 --- a/pelicun/tests/basic/test_demand_model.py +++ b/pelicun/tests/basic/test_demand_model.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -38,89 +37,113 @@ # Adam Zsarnóczay # John Vouvakis Manousakis -""" -These are unit and integration tests on the demand model of pelicun. -""" +"""These are unit and integration tests on the demand model of pelicun.""" from __future__ import annotations -from collections import defaultdict -import os + import tempfile import warnings +from collections import defaultdict from copy import deepcopy -import pytest +from pathlib import Path +from typing import TYPE_CHECKING + import numpy as np import pandas as pd +import pytest + +from pelicun.base import ensure_value +from pelicun.model.demand_model import ( + DemandModel, + _assemble_required_demand_data, + _get_required_demand_type, +) from pelicun.tests.basic.test_model import TestModelModule -from pelicun.model.demand_model import _get_required_demand_type -from pelicun.model.demand_model import _assemble_required_demand_data -# pylint: disable=unused-argument -# pylint: disable=missing-function-docstring -# pylint: disable=missing-class-docstring -# pylint: disable=missing-return-doc,missing-return-type-doc +if TYPE_CHECKING: + from pelicun.assessment import Assessment -class TestDemandModel(TestModelModule): +class TestDemandModel(TestModelModule): # noqa: PLR0904 @pytest.fixture - def demand_model(self, assessment_instance): + def demand_model(self, assessment_instance: Assessment) -> DemandModel: return deepcopy(assessment_instance.demand) @pytest.fixture - def demand_model_with_sample(self, assessment_instance): + def demand_model_with_sample( + self, assessment_instance: Assessment + ) -> DemandModel: mdl = assessment_instance.demand mdl.load_sample( 'pelicun/tests/basic/data/model/' 'test_DemandModel/load_sample/demand_sample_A.csv' ) - return deepcopy(mdl) + model_copy = deepcopy(mdl) + assert isinstance(model_copy, DemandModel) + return model_copy @pytest.fixture - def calibrated_demand_model(self, demand_model_with_sample): + def calibrated_demand_model( + self, demand_model_with_sample: DemandModel + ) -> DemandModel: config = { - "ALL": { - "DistributionFamily": "normal", - "AddUncertainty": 0.00, + 'ALL': { + 'DistributionFamily': 'normal', + 'AddUncertainty': 0.00, }, - "PID": { - "DistributionFamily": "lognormal", - "TruncateUpper": "0.06", + 'PID': { + 'DistributionFamily': 'lognormal', + 'TruncateUpper': '0.06', }, - "SA": { - "DistributionFamily": "empirical", + 'SA': { + 'DistributionFamily': 'empirical', }, } demand_model_with_sample.calibrate_model(config) - return deepcopy(demand_model_with_sample) + model_copy = deepcopy(demand_model_with_sample) + assert isinstance(model_copy, DemandModel) + return model_copy @pytest.fixture - def demand_model_with_sample_B(self, assessment_instance): + def demand_model_with_sample_b( + self, assessment_instance: Assessment + ) -> DemandModel: mdl = assessment_instance.demand mdl.load_sample( 'pelicun/tests/basic/data/model/' 'test_DemandModel/load_sample/demand_sample_B.csv' ) - return deepcopy(mdl) + model_copy = deepcopy(mdl) + assert isinstance(model_copy, DemandModel) + return model_copy @pytest.fixture - def demand_model_with_sample_C(self, assessment_instance): + def demand_model_with_sample_c( + self, assessment_instance: Assessment + ) -> DemandModel: mdl = assessment_instance.demand mdl.load_sample( 'pelicun/tests/basic/data/model/' 'test_DemandModel/load_sample/demand_sample_C.csv' ) - return deepcopy(mdl) + model_copy = deepcopy(mdl) + assert isinstance(model_copy, DemandModel) + return model_copy @pytest.fixture - def demand_model_with_sample_D(self, assessment_instance): + def demand_model_with_sample_d( + self, assessment_instance: Assessment + ) -> DemandModel: mdl = assessment_instance.demand mdl.load_sample( 'pelicun/tests/basic/data/model/' 'test_DemandModel/load_sample/demand_sample_D.csv' ) - return deepcopy(mdl) + model_copy = deepcopy(mdl) + assert isinstance(model_copy, DemandModel) + return model_copy - def test_init(self, demand_model): + def test_init(self, demand_model: DemandModel) -> None: assert demand_model.log assert demand_model.marginal_params is None @@ -130,19 +153,20 @@ def test_init(self, demand_model): assert demand_model._RVs is None assert demand_model.sample is None - def test_save_sample(self, demand_model_with_sample): + def test_save_sample(self, demand_model_with_sample: DemandModel) -> None: # instantiate a temporary directory in memory temp_dir = tempfile.mkdtemp() # save the sample there demand_model_with_sample.save_sample(f'{temp_dir}/temp.csv') - with open(f'{temp_dir}/temp.csv', 'r', encoding='utf-8') as f: + with Path(f'{temp_dir}/temp.csv').open(encoding='utf-8') as f: contents = f.read() assert contents == ( ',PFA-0-1,PFA-1-1,PID-1-1,SA_0.23-0-1\n' 'Units,inps2,inps2,rad,inps2\n' '0,158.62478,397.04389,0.02672,342.149\n' ) - res = demand_model_with_sample.save_sample(save_units=False) + res = demand_model_with_sample.save_sample() + assert isinstance(res, pd.DataFrame) assert res.to_dict() == { ('PFA', '0', '1'): {0: 158.62478}, ('PFA', '1', '1'): {0: 397.04389}, @@ -150,15 +174,19 @@ def test_save_sample(self, demand_model_with_sample): ('SA_0.23', '0', '1'): {0: 342.149}, } - def test_load_sample(self, demand_model_with_sample, demand_model_with_sample_B): + def test_load_sample( + self, + demand_model_with_sample: DemandModel, + demand_model_with_sample_b: DemandModel, + ) -> None: # retrieve the loaded sample and units - obtained_sample = demand_model_with_sample.sample - obtained_units = demand_model_with_sample.user_units + obtained_sample = ensure_value(demand_model_with_sample.sample) + obtained_units = ensure_value(demand_model_with_sample.user_units) - obtained_sample_2 = demand_model_with_sample_B.sample - obtained_units_2 = demand_model_with_sample_B.user_units + obtained_sample_2 = ensure_value(demand_model_with_sample_b.sample) + obtained_units_2 = ensure_value(demand_model_with_sample_b.user_units) - # demand_sample_A.csv and demand_sample_B.csv only differ in the + # demand_sample_A.csv and demand_sample_b.csv only differ in the # headers, where the first includes a tag for the hazard # level. Therefore, the two files are expected to result to the # same `obtained_sample` @@ -212,144 +240,187 @@ def test_load_sample(self, demand_model_with_sample, demand_model_with_sample_B) check_index_type=False, ) - def test_estimate_RID(self, demand_model_with_sample): - demands = demand_model_with_sample.sample['PID'] + def test_estimate_RID(self, demand_model_with_sample: DemandModel) -> None: + demands = ensure_value(demand_model_with_sample.sample)['PID'] params = {'yield_drift': 0.01} res = demand_model_with_sample.estimate_RID(demands, params) assert list(res.columns) == [('RID', '1', '1')] - assert ( + with pytest.raises(ValueError, match='Invalid method: `xyz`'): demand_model_with_sample.estimate_RID(demands, params, method='xyz') - is None + + def test_expand_sample_float( + self, demand_model_with_sample: DemandModel + ) -> None: + sample_before = ensure_value(demand_model_with_sample.sample).copy() + demand_model_with_sample.expand_sample('test_lab', 1.00, 'unitless') + sample_after = ensure_value(demand_model_with_sample.sample).copy() + pd.testing.assert_frame_equal( + sample_before, sample_after.drop('test_lab', axis=1) ) + assert sample_after.loc[0, ('test_lab', '0', '1')] == 1.0 + + def test_expand_sample_numpy( + self, demand_model_with_sample: DemandModel + ) -> None: + sample_before = ensure_value(demand_model_with_sample.sample).copy() + demand_model_with_sample.expand_sample('test_lab', 1.00, 'unitless') + sample_after = ensure_value(demand_model_with_sample.sample).copy() + pd.testing.assert_frame_equal( + sample_before, sample_after.drop('test_lab', axis=1) + ) + assert sample_after.loc[0, ('test_lab', '0', '1')] == 1.0 + + def test_expand_sample_error_no_sample(self, demand_model: DemandModel) -> None: + with pytest.raises( + ValueError, match='Demand model does not have a sample yet.' + ): + demand_model.expand_sample('test_lab', np.array((1.00,)), 'unitless') + + def test_expand_sample_error_wrong_shape( + self, demand_model_with_sample: DemandModel + ) -> None: + with pytest.raises(ValueError, match='Incompatible array length.'): + demand_model_with_sample.expand_sample( + 'test_lab', np.array((1.00, 1.00)), 'unitless' + ) def test_calibrate_model( - self, calibrated_demand_model, demand_model_with_sample_C - ): - assert calibrated_demand_model.marginal_params['Family'].to_list() == [ + self, + calibrated_demand_model: DemandModel, + ) -> None: + assert ensure_value(calibrated_demand_model.marginal_params)[ + 'Family' + ].to_list() == [ 'normal', 'normal', 'lognormal', 'empirical', ] assert ( - calibrated_demand_model.marginal_params.at[ + ensure_value(calibrated_demand_model.marginal_params).loc[ ('PID', '1', '1'), 'TruncateUpper' ] == 0.06 ) def test_calibrate_model_censoring( - self, calibrated_demand_model, demand_model_with_sample_C - ): + self, + demand_model_with_sample_c: DemandModel, + ) -> None: # with a config featuring censoring the RIDs config = { - "ALL": { - "DistributionFamily": "normal", - "AddUncertainty": 0.00, + 'ALL': { + 'DistributionFamily': 'normal', + 'AddUncertainty': 0.00, }, - "PID": { - "DistributionFamily": "lognormal", - "CensorUpper": "0.05", + 'PID': { + 'DistributionFamily': 'lognormal', + 'CensorUpper': '0.05', }, } - demand_model_with_sample_C.calibrate_model(config) + demand_model_with_sample_c.calibrate_model(config) def test_calibrate_model_truncation( - self, calibrated_demand_model, demand_model_with_sample_C - ): + self, + demand_model_with_sample_c: DemandModel, + ) -> None: # with a config that specifies a truncation limit smaller than # the samples config = { - "ALL": { - "DistributionFamily": "normal", - "AddUncertainty": 0.00, + 'ALL': { + 'DistributionFamily': 'normal', + 'AddUncertainty': 0.00, }, - "PID": { - "DistributionFamily": "lognormal", - "TruncateUpper": "0.04", + 'PID': { + 'DistributionFamily': 'lognormal', + 'TruncateUpper': '0.04', }, } - demand_model_with_sample_C.calibrate_model(config) + demand_model_with_sample_c.calibrate_model(config) def test_save_load_model_with_empirical( - self, calibrated_demand_model, assessment_instance - ): - + self, calibrated_demand_model: DemandModel, assessment_instance: Assessment + ) -> None: # a model that has empirical marginal parameters temp_dir = tempfile.mkdtemp() calibrated_demand_model.save_model(f'{temp_dir}/temp') - assert os.path.exists(f'{temp_dir}/temp_marginals.csv') - assert os.path.exists(f'{temp_dir}/temp_empirical.csv') - assert os.path.exists(f'{temp_dir}/temp_correlation.csv') + assert Path(f'{temp_dir}/temp_marginals.csv').exists() + assert Path(f'{temp_dir}/temp_empirical.csv').exists() + assert Path(f'{temp_dir}/temp_correlation.csv').exists() # Load model to a different DemandModel instance to verify new_demand_model = assessment_instance.demand new_demand_model.load_model(f'{temp_dir}/temp') pd.testing.assert_frame_equal( - calibrated_demand_model.marginal_params, - new_demand_model.marginal_params, + ensure_value(calibrated_demand_model.marginal_params), + ensure_value(new_demand_model.marginal_params), atol=1e-4, check_index_type=False, check_column_type=False, ) pd.testing.assert_frame_equal( - calibrated_demand_model.correlation, - new_demand_model.correlation, + ensure_value(calibrated_demand_model.correlation), + ensure_value(new_demand_model.correlation), atol=1e-4, check_index_type=False, check_column_type=False, ) pd.testing.assert_frame_equal( - calibrated_demand_model.empirical_data, - new_demand_model.empirical_data, + ensure_value(calibrated_demand_model.empirical_data), + ensure_value(new_demand_model.empirical_data), atol=1e-4, check_index_type=False, check_column_type=False, ) def test_save_load_model_without_empirical( - self, demand_model_with_sample_C, assessment_instance - ): + self, + demand_model_with_sample_c: DemandModel, + assessment_instance: Assessment, + ) -> None: # a model that does not have empirical marginal parameters temp_dir = tempfile.mkdtemp() config = { - "ALL": { - "DistributionFamily": "normal", - "AddUncertainty": 0.00, + 'ALL': { + 'DistributionFamily': 'normal', + 'AddUncertainty': 0.00, }, - "PID": { - "DistributionFamily": "lognormal", - "TruncateUpper": "0.04", + 'PID': { + 'DistributionFamily': 'lognormal', + 'TruncateUpper': '0.04', }, } - demand_model_with_sample_C.calibrate_model(config) - demand_model_with_sample_C.save_model(f'{temp_dir}/temp') - assert os.path.exists(f'{temp_dir}/temp_marginals.csv') - assert os.path.exists(f'{temp_dir}/temp_correlation.csv') + demand_model_with_sample_c.calibrate_model(config) + demand_model_with_sample_c.save_model(f'{temp_dir}/temp') + assert Path(f'{temp_dir}/temp_marginals.csv').exists() + assert Path(f'{temp_dir}/temp_correlation.csv').exists() # Load model to a different DemandModel instance to verify new_demand_model = assessment_instance.demand new_demand_model.load_model(f'{temp_dir}/temp') pd.testing.assert_frame_equal( - demand_model_with_sample_C.marginal_params, - new_demand_model.marginal_params, + ensure_value(demand_model_with_sample_c.marginal_params), + ensure_value(new_demand_model.marginal_params), ) pd.testing.assert_frame_equal( - demand_model_with_sample_C.correlation, new_demand_model.correlation + ensure_value(demand_model_with_sample_c.correlation), + ensure_value(new_demand_model.correlation), ) - assert demand_model_with_sample_C.empirical_data is None + assert demand_model_with_sample_c.empirical_data is None assert new_demand_model.empirical_data is None - def test_generate_sample_exceptions(self, demand_model): + def test_generate_sample_exceptions(self, demand_model: DemandModel) -> None: # generating a sample from a non calibrated model should fail - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match='Model parameters have not been specified' + ): demand_model.generate_sample( - {"SampleSize": 3, 'PreserveRawOrder': False} + {'SampleSize': 3, 'PreserveRawOrder': False} ) - def test_generate_sample(self, calibrated_demand_model): + def test_generate_sample(self, calibrated_demand_model: DemandModel) -> None: calibrated_demand_model.generate_sample( - {"SampleSize": 3, 'PreserveRawOrder': False} + {'SampleSize': 3, 'PreserveRawOrder': False} ) # get the generated demand sample @@ -397,7 +468,9 @@ def test_generate_sample(self, calibrated_demand_model): check_index_type=False, ) - def test_generate_sample_with_demand_cloning(self, assessment_instance): + def test_generate_sample_with_demand_cloning( + self, assessment_instance: Assessment + ) -> None: # # used for debugging # assessment_instance = assessment.Assessment() @@ -412,8 +485,8 @@ def test_generate_sample_with_demand_cloning(self, assessment_instance): ) demand_model.calibrate_model( { - "ALL": { - "DistributionFamily": "lognormal", + 'ALL': { + 'DistributionFamily': 'lognormal', }, } ) @@ -431,12 +504,12 @@ def test_generate_sample_with_demand_cloning(self, assessment_instance): ) assert len(w) == 1 assert ( - "The demand cloning configuration lists columns " + 'The demand cloning configuration lists columns ' "that are not present in the original demand sample's " "columns: ['not_present']." ) in str(w[0].message) # we'll just get a warning for the `not_present` entry - assert demand_model.sample.columns.to_list() == [ + assert ensure_value(demand_model.sample).columns.to_list() == [ ('PGA', '0', '1'), ('PGV', '0', '1'), ('PGV', '0', '2'), @@ -449,12 +522,14 @@ def test_generate_sample_with_demand_cloning(self, assessment_instance): ('PGV', '2', '3'), ] assert np.array_equal( - demand_model.sample[('PGV', '0', '1')].values, - demand_model.sample[('PGV', '0', '3')].values, + demand_model.sample['PGV', '0', '1'].values, # type: ignore + demand_model.sample['PGV', '0', '3'].values, # type: ignore ) # exceptions # Duplicate entries in demand cloning configuration - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match='Duplicate entries in demand cloning configuration.' + ): demand_model.generate_sample( { 'SampleSize': 1000, @@ -466,8 +541,9 @@ def test_generate_sample_with_demand_cloning(self, assessment_instance): } ) - def test__get_required_demand_type(self, assessment_instance): - + def test__get_required_demand_type( + self, assessment_instance: Assessment + ) -> None: # Simple case: single demand damage_model = assessment_instance.damage cmp_set = {'testing.component'} @@ -483,7 +559,7 @@ def test__get_required_demand_type(self, assessment_instance): ).T.rename_axis(index=['cmp', 'loc', 'dir', 'uid']) demand_offset = {'PFA': 0} required = _get_required_demand_type( - damage_model.ds_model.damage_params, pgb, demand_offset + ensure_value(damage_model.ds_model.damage_params), pgb, demand_offset ) expected = defaultdict( list, @@ -506,20 +582,22 @@ def test__get_required_demand_type(self, assessment_instance): ).T.rename_axis(index=['cmp', 'loc', 'dir', 'uid']) demand_offset = {'PFA': 0} required = _get_required_demand_type( - damage_model.ds_model.damage_params, pgb, demand_offset + ensure_value(damage_model.ds_model.damage_params), pgb, demand_offset ) expected = defaultdict( list, { - (('PID-1-1', 'PFA-1-1'), 'sqrt(X1^2+X2^2)'): [ + (('PID-1-1', 'PFA-1-1'), 'sqrt(X1^2+X2^2)'): [ # type: ignore ('testing.component', '1', '1', '1') ] }, ) assert required == expected - def test__assemble_required_demand_data(self, assessment_instance): - + def test__assemble_required_demand_data( + self, assessment_instance: Assessment + ) -> None: + # Utility demand case: two demands are required damage_model = assessment_instance.damage cmp_set = {'testing.component'} damage_model.load_model_parameters( @@ -545,7 +623,9 @@ def test__assemble_required_demand_data(self, assessment_instance): } ) demand_data = _assemble_required_demand_data( - required_edps, nondirectional_multipliers, demand_sample + required_edps, # type: ignore + nondirectional_multipliers, + demand_sample, ) expected = { (('PID-1-1', 'PFA-1-1'), 'sqrt(X1^2+X2^2)'): np.array( diff --git a/pelicun/tests/basic/test_file_io.py b/pelicun/tests/basic/test_file_io.py index 93850b70a..06444a707 100644 --- a/pelicun/tests/basic/test_file_io.py +++ b/pelicun/tests/basic/test_file_io.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -38,58 +37,57 @@ # Adam Zsarnóczay # John Vouvakis Manousakis -""" -These are unit and integration tests on the file_io module of pelicun. -""" +"""These are unit and integration tests on the file_io module of pelicun.""" from __future__ import annotations + import tempfile -import os -import pytest +from pathlib import Path + import numpy as np import pandas as pd -from pelicun import file_io -from pelicun import base -from pelicun.warnings import PelicunWarning +import pytest +from pelicun import base, file_io +from pelicun.pelicun_warnings import PelicunWarning # The tests maintain the order of definitions of the `file_io.py` file. -def test_save_to_csv(): +def test_save_to_csv() -> None: # Test saving with orientation 0 - data = pd.DataFrame({"A": [1e-3, 2e-3, 3e-3], "B": [4e-3, 5e-3, 6e-3]}) - units = pd.Series(["meters", "meters"], index=["A", "B"]) - unit_conversion_factors = {"meters": 0.001} + data = pd.DataFrame({'A': [1e-3, 2e-3, 3e-3], 'B': [4e-3, 5e-3, 6e-3]}) + units = pd.Series(['meters', 'meters'], index=['A', 'B']) + unit_conversion_factors = {'meters': 0.001} # Save to a temporary file with tempfile.TemporaryDirectory() as tmpdir: - filepath = os.path.join(tmpdir, 'foo.csv') + filepath = Path(tmpdir) / 'foo.csv' file_io.save_to_csv( data, filepath, units, unit_conversion_factors, orientation=0 ) - assert os.path.isfile(filepath) + assert Path(filepath).is_file() # Check that the file contains the expected data - with open(filepath, 'r', encoding='utf-8') as f: + with Path(filepath).open(encoding='utf-8') as f: contents = f.read() assert contents == ( ',A,B\n0,meters,meters\n0,1.0,4.0' '\n1,2.0,5.0\n2,3.0,6.0\n' ) # Test saving with orientation 1 - data = pd.DataFrame({"A": [1e-3, 2e-3, 3e-3], "B": [4e-3, 5e-3, 6e-3]}) - units = pd.Series(["meters", "meters"], index=["A", "B"]) - unit_conversion_factors = {"meters": 0.001} + data = pd.DataFrame({'A': [1e-3, 2e-3, 3e-3], 'B': [4e-3, 5e-3, 6e-3]}) + units = pd.Series(['meters', 'meters'], index=['A', 'B']) + unit_conversion_factors = {'meters': 0.001} # Save to a temporary file with tempfile.TemporaryDirectory() as tmpdir: - filepath = os.path.join(tmpdir, 'bar.csv') + filepath = Path(tmpdir) / 'bar.csv' file_io.save_to_csv( data, filepath, units, unit_conversion_factors, orientation=1 ) - assert os.path.isfile(filepath) + assert Path(filepath).is_file() # Check that the file contains the expected data - with open(filepath, 'r', encoding='utf-8') as f: + with Path(filepath).open(encoding='utf-8') as f: contents = f.read() assert contents == ( ',0,A,B\n0,,0.001,0.004\n1,,0.002,' '0.005\n2,,0.003,0.006\n' @@ -99,38 +97,40 @@ def test_save_to_csv(): # edge cases # - data = pd.DataFrame({"A": [1e-3, 2e-3, 3e-3], "B": [4e-3, 5e-3, 6e-3]}) - units = pd.Series(["meters", "meters"], index=["A", "B"]) + data = pd.DataFrame({'A': [1e-3, 2e-3, 3e-3], 'B': [4e-3, 5e-3, 6e-3]}) + units = pd.Series(['meters', 'meters'], index=['A', 'B']) # units given, without unit conversion factors - unit_conversion_factors = None - with pytest.raises(ValueError): - with tempfile.TemporaryDirectory() as tmpdir: - filepath = os.path.join(tmpdir, 'foo.csv') - file_io.save_to_csv( - data, filepath, units, unit_conversion_factors, orientation=0 - ) + filepath = Path(tmpdir) / 'foo.csv' + with pytest.raises( + ValueError, + match='When `units` is not None, `unit_conversion_factors` must be provided.', + ), tempfile.TemporaryDirectory() as tmpdir: + file_io.save_to_csv( + data, filepath, units, unit_conversion_factors=None, orientation=0 + ) - unit_conversion_factors = {"meters": 0.001} + unit_conversion_factors = {'meters': 0.001} # not csv extension - with pytest.raises(ValueError): - with tempfile.TemporaryDirectory() as tmpdir: - filepath = os.path.join(tmpdir, 'foo.xyz') - file_io.save_to_csv( - data, filepath, units, unit_conversion_factors, orientation=0 - ) + filepath = Path(tmpdir) / 'foo.xyz' + with pytest.raises( + ValueError, + match=('Please use the `.csv` file extension. Received file name is '), + ), tempfile.TemporaryDirectory() as tmpdir: + file_io.save_to_csv( + data, filepath, units, unit_conversion_factors, orientation=0 + ) # no data, log a complaint mylogger = base.Logger( - verbose=True, log_show_ms=False, log_file=None, print_log=True + log_file=None, verbose=True, log_show_ms=False, print_log=True ) - data = None with tempfile.TemporaryDirectory() as tmpdir: - filepath = os.path.join(tmpdir, 'foo.csv') + filepath = Path(tmpdir) / 'foo.csv' with pytest.warns(PelicunWarning) as record: file_io.save_to_csv( - data, + None, filepath, units, unit_conversion_factors, @@ -140,10 +140,13 @@ def test_save_to_csv(): assert 'Data was empty, no file saved.' in str(record.list[0].message) -def test_substitute_default_path(): +def test_substitute_default_path() -> None: prior_path = file_io.base.pelicun_path - file_io.base.pelicun_path = 'some_path' - input_paths = ['PelicunDefault/data/file1.txt', '/data/file2.txt'] + file_io.base.pelicun_path = Path('some_path') + input_paths: list[str | pd.DataFrame] = [ + 'PelicunDefault/data/file1.txt', + '/data/file2.txt', + ] expected_paths = [ 'some_path/resources/SimCenterDBDL/data/file1.txt', '/data/file2.txt', @@ -153,24 +156,24 @@ def test_substitute_default_path(): file_io.base.pelicun_path = prior_path -def test_load_data(): +def test_load_data() -> None: # test loading data with orientation 0 filepath = 'pelicun/tests/basic/data/file_io/test_load_data/units.csv' - unit_conversion_factors = {"inps2": 0.0254, "rad": 1.00} + unit_conversion_factors = {'inps2': 0.0254, 'rad': 1.00} data = file_io.load_data(filepath, unit_conversion_factors) - assert np.array_equal(data.index.values, np.array(range(6))) - assert data.shape == (6, 19) - assert isinstance(data.columns, pd.core.indexes.multi.MultiIndex) - assert data.columns.nlevels == 4 + assert np.array_equal(data.index.values, np.array(range(6))) # type: ignore + assert data.shape == (6, 19) # type: ignore + assert isinstance(data.columns, pd.core.indexes.multi.MultiIndex) # type: ignore + assert data.columns.nlevels == 4 # type: ignore _, units = file_io.load_data( filepath, unit_conversion_factors, return_units=True ) for item in unit_conversion_factors: - assert item in units.unique() + assert item in units.unique() # type: ignore filepath = 'pelicun/tests/basic/data/file_io/test_load_data/no_units.csv' data_nounits = file_io.load_data(filepath, {}) @@ -187,7 +190,7 @@ def test_load_data(): # with convert=None filepath = 'pelicun/tests/basic/data/file_io/test_load_data/orient_1_units.csv' - unit_conversion_factors = {"g": 1.00, "rad": 1.00} + unit_conversion_factors = {'g': 1.00, 'rad': 1.00} data = file_io.load_data( filepath, unit_conversion_factors, orientation=1, reindex=False ) @@ -199,7 +202,7 @@ def test_load_data(): data = file_io.load_data( filepath, unit_conversion_factors, orientation=1, reindex=True ) - assert np.array_equal(data.index.values, np.array(range(10))) + assert np.array_equal(data.index.values, np.array(range(10))) # type: ignore # # edge cases @@ -209,8 +212,11 @@ def test_load_data(): with pytest.raises(FileNotFoundError): file_io.load_from_file('/') # exception: not a .csv file - with pytest.raises(ValueError): - file_io.load_from_file('pelicun/db.py') + with pytest.raises( + ValueError, + match='Unexpected file type received when trying to load from csv', + ): + file_io.load_from_file('pelicun/base.py') if __name__ == '__main__': diff --git a/pelicun/tests/basic/test_loss_model.py b/pelicun/tests/basic/test_loss_model.py index 0277f0532..2f7467144 100644 --- a/pelicun/tests/basic/test_loss_model.py +++ b/pelicun/tests/basic/test_loss_model.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -38,44 +37,47 @@ # Adam Zsarnóczay # John Vouvakis Manousakis -""" -These are unit and integration tests on the loss model of pelicun. -""" +"""These are unit and integration tests on the loss model of pelicun.""" from __future__ import annotations -from itertools import product + +import re from copy import deepcopy -import pytest +from itertools import product +from typing import TYPE_CHECKING + import numpy as np import pandas as pd -from pelicun import model -from pelicun import uq +import pytest + +from pelicun import model, uq +from pelicun.base import ensure_value +from pelicun.model.loss_model import ( + LossModel, + RepairModel_DS, + RepairModel_LF, + _is_for_ds_model, + _is_for_lf_model, +) +from pelicun.pelicun_warnings import PelicunWarning from pelicun.tests.basic.test_pelicun_model import TestPelicunModel -from pelicun.model.loss_model import LossModel -from pelicun.model.loss_model import RepairModel_Base -from pelicun.model.loss_model import RepairModel_DS -from pelicun.model.loss_model import RepairModel_LF -from pelicun.model.loss_model import _is_for_ds_model -from pelicun.model.loss_model import _is_for_lf_model -from pelicun.warnings import PelicunWarning -# pylint: disable=missing-function-docstring -# pylint: disable=missing-class-docstring -# pylint: disable=missing-return-doc,missing-return-type-doc +if TYPE_CHECKING: + from pelicun.assessment import Assessment + from pelicun.model.asset_model import AssetModel class TestLossModel(TestPelicunModel): - @pytest.fixture - def loss_model(self, assessment_instance): + def loss_model(self, assessment_instance: Assessment) -> LossModel: return deepcopy(assessment_instance.loss) @pytest.fixture - def asset_model_empty(self, assessment_instance): + def asset_model_empty(self, assessment_instance: Assessment) -> AssetModel: return deepcopy(assessment_instance.asset) @pytest.fixture - def asset_model_A(self, asset_model_empty): + def asset_model_a(self, asset_model_empty: AssetModel) -> AssetModel: asset = deepcopy(asset_model_empty) asset.cmp_marginal_params = pd.DataFrame( { @@ -94,7 +96,7 @@ def asset_model_A(self, asset_model_empty): return asset @pytest.fixture - def loss_model_with_ones(self, assessment_instance): + def loss_model_with_ones(self, assessment_instance: Assessment) -> LossModel: loss_model = assessment_instance.loss # add artificial values to the samples @@ -117,15 +119,13 @@ def loss_model_with_ones(self, assessment_instance): ('uid1', 'uid2'), ): data_ds[ - ( - decision_variable, - consequence, - component, - damage_state, - location, - direction, - uid, - ) + decision_variable, + consequence, + component, + damage_state, + location, + direction, + uid, ] = [1.00, 1.00, 1.00] loss_model.ds_model.sample = pd.DataFrame(data_ds).rename_axis( columns=['dv', 'loss', 'dmg', 'ds', 'loc', 'dir', 'uid'] @@ -147,14 +147,12 @@ def loss_model_with_ones(self, assessment_instance): ('uid1', 'uid2'), ): data_lf[ - ( - decision_variable, - consequence, - component, - location, - direction, - uid, - ) + decision_variable, + consequence, + component, + location, + direction, + uid, ] = [1.00, 1.00, 1.00] loss_model.lf_model.sample = pd.DataFrame(data_lf).rename_axis( columns=['dv', 'loss', 'dmg', 'loc', 'dir', 'uid'] @@ -162,43 +160,47 @@ def loss_model_with_ones(self, assessment_instance): return loss_model - def test___init__(self, loss_model): + def test___init__(self, loss_model: LossModel) -> None: assert loss_model.log assert loss_model.ds_model with pytest.raises(AttributeError): - loss_model.xyz = 123 + loss_model.xyz = 123 # type: ignore assert loss_model.ds_model.loss_params is None assert loss_model.ds_model.sample is None assert len(loss_model._loss_models) == 2 - def test_decision_variables(self, loss_model): - dvs = ('Carbon', 'Cost', 'Energy', 'Time') + def test_decision_variables(self, loss_model: LossModel) -> None: + dvs = ('Cost', 'Time') assert loss_model.decision_variables == dvs assert loss_model.ds_model.decision_variables == dvs assert loss_model.lf_model.decision_variables == dvs - def test_add_loss_map(self, loss_model, asset_model_A): - - loss_model._asmnt.asset = asset_model_A + def test_add_loss_map( + self, loss_model: LossModel, asset_model_a: AssetModel + ) -> None: + loss_model._asmnt.asset = asset_model_a - loss_map = loss_map = pd.DataFrame( + loss_map = pd.DataFrame( { 'Repair': ['consequence.A', 'consequence.B'], }, index=['cmp.A', 'cmp.B'], ) loss_model.add_loss_map(loss_map) - pd.testing.assert_frame_equal(loss_model._loss_map, loss_map) + pd.testing.assert_frame_equal(ensure_value(loss_model._loss_map), loss_map) for contained_model in loss_model._loss_models: - pd.testing.assert_frame_equal(contained_model._loss_map, loss_map) - - def test_load_model_parameters(self, loss_model, asset_model_A): + pd.testing.assert_frame_equal( + ensure_value(contained_model.loss_map), loss_map + ) - loss_model._asmnt.asset = asset_model_A + def test_load_model_parameters( + self, loss_model: LossModel, asset_model_a: AssetModel + ) -> None: + loss_model._asmnt.asset = asset_model_a loss_model.decision_variables = ('my_RV',) - loss_map = loss_map = pd.DataFrame( + loss_map = pd.DataFrame( { 'Repair': ['consequence.A', 'consequence.B', 'consequence.F'], }, @@ -238,23 +240,23 @@ def test_load_model_parameters(self, loss_model, asset_model_A): ) # assert len(record) == 1 - # TODO: re-enable the line above once we address other + # TODO(JVM): re-enable the line above once we address other # warnings, and change indexing to [0] below. assert ( - "The loss model does not provide loss information " - "for the following component(s) in the asset " + 'The loss model does not provide loss information ' + 'for the following component(s) in the asset ' "model: [('consequence.F', 'my_RV')]." ) in str(record[-1].message) - def test__loss_models(self, loss_model): + def test__loss_models(self, loss_model: LossModel) -> None: models = loss_model._loss_models assert len(models) == 2 assert isinstance(models[0], RepairModel_DS) assert isinstance(models[1], RepairModel_LF) - def test__loss_map(self, loss_model): - loss_map = loss_map = pd.DataFrame( + def test__loss_map(self, loss_model: LossModel) -> None: + loss_map = pd.DataFrame( { 'Repair': ['consequence_A', 'consequence_B'], }, @@ -263,11 +265,13 @@ def test__loss_map(self, loss_model): # test setter loss_model._loss_map = loss_map # test getter - pd.testing.assert_frame_equal(loss_model._loss_map, loss_map) + pd.testing.assert_frame_equal(ensure_value(loss_model._loss_map), loss_map) for contained_model in loss_model._loss_models: - pd.testing.assert_frame_equal(contained_model._loss_map, loss_map) + pd.testing.assert_frame_equal( + ensure_value(contained_model.loss_map), loss_map + ) - def test__missing(self, loss_model): + def test__missing(self, loss_model: LossModel) -> None: missing = { ('missing.component', 'Time'), ('missing.component', 'Energy'), @@ -277,9 +281,11 @@ def test__missing(self, loss_model): # test getter assert loss_model._missing == missing for contained_model in loss_model._loss_models: - assert contained_model._missing == missing + assert contained_model.missing == missing - def test__ensure_loss_parameter_availability(self, assessment_instance): + def test__ensure_loss_parameter_availability( + self, assessment_instance: Assessment + ) -> None: loss_model = LossModel(assessment_instance) # Only consider `DecisionVariableXYZ` @@ -289,7 +295,7 @@ def test__ensure_loss_parameter_availability(self, assessment_instance): # C, D should be in the lf model # E should be missing - loss_map = loss_map = pd.DataFrame( + loss_map = pd.DataFrame( { 'Repair': [f'consequence_{x}' for x in ('A', 'B', 'C', 'D', 'E')], }, @@ -321,22 +327,25 @@ def test__ensure_loss_parameter_availability(self, assessment_instance): assert missing == {('consequence_E', 'DecisionVariableXYZ')} assert len(record) == 1 assert ( - "The loss model does not provide loss information " - "for the following component(s) in the asset model: " + 'The loss model does not provide loss information ' + 'for the following component(s) in the asset model: ' "[('consequence_E', 'DecisionVariableXYZ')]" ) in str(record[0].message) - def test_aggregate_losses_when_no_loss(self, assessment_instance): - + def test_aggregate_losses_when_no_loss( + self, assessment_instance: Assessment + ) -> None: # tests that aggregate losses works when there is no loss. loss_model = LossModel(assessment_instance) + loss_model.decision_variables = ('Cost', 'Time', 'Carbon', 'Energy') df_agg = loss_model.aggregate_losses() + assert isinstance(df_agg, pd.DataFrame) pd.testing.assert_frame_equal( df_agg, pd.DataFrame( { - 'repair_carbon': 0.0, 'repair_cost': 0.00, + 'repair_carbon': 0.0, 'repair_energy': 0.00, 'repair_time-sequential': 0.00, 'repair_time-parallel': 0.00, @@ -345,8 +354,9 @@ def test_aggregate_losses_when_no_loss(self, assessment_instance): ), ) - def test__apply_consequence_scaling(self, loss_model_with_ones): - + def test__apply_consequence_scaling( + self, loss_model_with_ones: LossModel + ) -> None: # When only `dv` is provided scaling_conditions = {'dv': 'Cost'} scaling_factor = 2.00 @@ -356,6 +366,7 @@ def test__apply_consequence_scaling(self, loss_model_with_ones): ) for loss_model in loss_model_with_ones._loss_models: + assert loss_model.sample is not None mask = loss_model.sample.columns.get_level_values('dv') == 'Cost' assert np.all(loss_model.sample.iloc[:, mask] == 2.00) assert np.all(loss_model.sample.iloc[:, ~mask] == 1.00) @@ -368,15 +379,17 @@ def test__apply_consequence_scaling(self, loss_model_with_ones): ) for loss_model in loss_model_with_ones._loss_models: - mask = np.full(len(loss_model.sample.columns), True) + assert loss_model.sample is not None + mask = np.full(len(loss_model.sample.columns), fill_value=True) mask &= loss_model.sample.columns.get_level_values('dv') == 'Carbon' mask &= loss_model.sample.columns.get_level_values('loc') == '1' mask &= loss_model.sample.columns.get_level_values('uid') == 'uid2' assert np.all(loss_model.sample.iloc[:, mask] == 2.00) assert np.all(loss_model.sample.iloc[:, ~mask] == 1.00) - def test_aggregate_losses_combination(self, assessment_instance): - + def test_aggregate_losses_combination( + self, assessment_instance: Assessment + ) -> None: # The test sets up a very simple loss calculation from # scratch, only defining essential parameters. @@ -389,15 +402,15 @@ def test_aggregate_losses_combination(self, assessment_instance): }, index=['Units', 'Theta_0'], ).T - perfect_CORR = pd.DataFrame( + perfect_corr = pd.DataFrame( np.ones((2, 2)), columns=demand_marginal_parameters.index, index=demand_marginal_parameters.index, ) assessment_instance.demand.load_model( - {'marginals': demand_marginal_parameters, 'correlation': perfect_CORR} + {'marginals': demand_marginal_parameters, 'correlation': perfect_corr} ) - assessment_instance.demand.generate_sample({"SampleSize": sample_size}) + assessment_instance.demand.generate_sample({'SampleSize': sample_size}) # asset assessment_instance.asset.cmp_marginal_params = pd.DataFrame( @@ -433,7 +446,7 @@ def test_aggregate_losses_combination(self, assessment_instance): assessment_instance.loss.calculate() # individual losses - l1, l2 = assessment_instance.loss.lf_model.sample.iloc[0, :] + l1, l2 = ensure_value(assessment_instance.loss.lf_model.sample).iloc[0, :] # combined loss, result of interpolation l_comb = 0.904 @@ -444,7 +457,7 @@ def test_aggregate_losses_combination(self, assessment_instance): ), index_col=None, header=None, - ).values + ).to_numpy() loss_combination = { 'Cost': { ('wind.comp', 'flood.comp'): combination_array, @@ -454,6 +467,7 @@ def test_aggregate_losses_combination(self, assessment_instance): agg_df, _ = assessment_instance.loss.aggregate_losses( loss_combination=loss_combination, future=True ) + assert isinstance(agg_df, pd.DataFrame) pd.testing.assert_frame_equal( agg_df, pd.DataFrame([l_comb] * 5, columns=['repair_cost']) ) @@ -464,33 +478,40 @@ def test_aggregate_losses_combination(self, assessment_instance): assert l2 == combination_array[0, 4] assert combination_array[8, 0] <= l1 <= combination_array[9, 0] - def test_aggregate_losses_thresholds(self, loss_model_with_ones): - + def test_aggregate_losses_thresholds( + self, loss_model_with_ones: LossModel + ) -> None: # Row 0 has the value of 1.0 in all columns. # Adjust rows 1 and 2 to have the values 2.0 and 3.0, for # testing. + assert loss_model_with_ones.ds_model.sample is not None + assert loss_model_with_ones.lf_model.sample is not None loss_model_with_ones.decision_variables = ('Cost', 'Carbon') + loss_model_with_ones.dv_units = {'Cost': 'USD_2011', 'Carbon': 'kg'} loss_model_with_ones.ds_model.sample.iloc[1, :] = 2.00 loss_model_with_ones.ds_model.sample.iloc[2, :] = 3.00 loss_model_with_ones.lf_model.sample.iloc[1, :] = 2.00 loss_model_with_ones.lf_model.sample.iloc[2, :] = 3.00 # Instantiate a RandomVariableRegistry to pass as an argument # to the method. - RV_reg = uq.RandomVariableRegistry(loss_model_with_ones._asmnt.options.rng) + rv_reg = uq.RandomVariableRegistry(loss_model_with_ones._asmnt.options.rng) # Add a threshold for `Cost` - RV_reg.add_RV( - uq.rv_class_map('deterministic')(name='Cost', theta=np.array((400.00,))) + rv_reg.add_RV( + uq.rv_class_map('deterministic')(name='Cost', theta=np.array((400.00,))) # type: ignore ) # Add a threshold for `Carbon` - RV_reg.add_RV( + rv_reg.add_RV( uq.rv_class_map('deterministic')( - name='Carbon', theta=np.array((100.00,)) + name='Carbon', + theta=np.array((100.00,)), # type: ignore ) ) df_agg, exceedance_bool_df = loss_model_with_ones.aggregate_losses( - replacement_configuration=(RV_reg, {'Cost': 0.50, 'Carbon': 1.00}), + replacement_configuration=(rv_reg, {'Cost': 0.50, 'Carbon': 1.00}), future=True, ) + assert isinstance(df_agg, pd.DataFrame) + assert isinstance(exceedance_bool_df, pd.DataFrame) df_agg_expected = pd.DataFrame( { 'repair_carbon': [96.00, 100.00, 100.00], @@ -505,8 +526,7 @@ def test_aggregate_losses_thresholds(self, loss_model_with_ones): exceedance_bool_df, exceedance_bool_df_expected ) - def test_consequence_scaling(self, loss_model_with_ones): - + def test_consequence_scaling(self, loss_model_with_ones: LossModel) -> None: loss_model_with_ones.consequence_scaling( 'pelicun/tests/basic/data/model/test_LossModel/scaling_specification.csv' ) @@ -527,9 +547,10 @@ def test_consequence_scaling(self, loss_model_with_ones): .set_index(['dv', 'loss', 'dmg', 'ds', 'loc', 'dir', 'uid']) .T.astype(float) ) - expected_ds.index = pd.RangeIndex(range(len(expected_ds))) + expected_ds.index = pd.RangeIndex(range(len(expected_ds))) # type: ignore pd.testing.assert_frame_equal( - loss_model_with_ones.ds_model.sample, expected_ds + loss_model_with_ones.ds_model.sample, # type: ignore + expected_ds, ) expected_lf = ( @@ -547,48 +568,52 @@ def test_consequence_scaling(self, loss_model_with_ones): .set_index(['dv', 'loss', 'dmg', 'loc', 'dir', 'uid']) .T.astype(float) ) - expected_lf.index = pd.RangeIndex(range(len(expected_lf))) + expected_lf.index = pd.RangeIndex(range(len(expected_lf))) # type: ignore pd.testing.assert_frame_equal( - loss_model_with_ones.lf_model.sample, expected_lf + loss_model_with_ones.lf_model.sample, # type: ignore + expected_lf, ) class TestRepairModel_Base(TestPelicunModel): - def test___init__(self, assessment_instance): - repair_model = RepairModel_Base(assessment_instance) + def test___init__(self, assessment_instance: Assessment) -> None: + repair_model = RepairModel_DS(assessment_instance) with pytest.raises(AttributeError): - # pylint: disable=assigning-non-slot - repair_model.xyz = 123 + repair_model.xyz = 123 # type: ignore - def test__drop_unused_loss_parameters(self, assessment_instance): - base_model = RepairModel_Base(assessment_instance) - loss_map = loss_map = pd.DataFrame( + def test_drop_unused_loss_parameters( + self, assessment_instance: Assessment + ) -> None: + base_model = RepairModel_DS(assessment_instance) + loss_map = pd.DataFrame( { 'Repair': ['consequence_A', 'consequence_B'], }, index=['cmp_A', 'cmp_B'], ) # without loss_params, it should do nothing - base_model._drop_unused_loss_parameters(loss_map) + base_model.drop_unused_loss_parameters(loss_map) base_model.loss_params = pd.DataFrame( index=[f'consequence_{x}' for x in ('A', 'B', 'C', 'D')] ) - base_model._drop_unused_loss_parameters(loss_map) + base_model.drop_unused_loss_parameters(loss_map) pd.testing.assert_frame_equal( base_model.loss_params, pd.DataFrame(index=[f'consequence_{x}' for x in ('A', 'B')]), ) - def test__remove_incomplete_components(self, assessment_instance): - base_model = RepairModel_Base(assessment_instance) + def test__remove_incomplete_components( + self, assessment_instance: Assessment + ) -> None: + base_model = RepairModel_DS(assessment_instance) # without loss_params, it should do nothing - base_model._remove_incomplete_components() + base_model.remove_incomplete_components() # without incomplete, it should do nothing loss_params = pd.DataFrame( index=[f'consequence_{x}' for x in ('A', 'B', 'C', 'D')] ) base_model.loss_params = loss_params - base_model._remove_incomplete_components() + base_model.remove_incomplete_components() pd.testing.assert_frame_equal( base_model.loss_params, loss_params, @@ -598,7 +623,7 @@ def test__remove_incomplete_components(self, assessment_instance): index=[f'consequence_{x}' for x in ('A', 'B', 'C', 'D')], ) # Now entry D should be gone - base_model._remove_incomplete_components() + base_model.remove_incomplete_components() pd.testing.assert_frame_equal( base_model.loss_params, pd.DataFrame( @@ -607,14 +632,16 @@ def test__remove_incomplete_components(self, assessment_instance): ), ) - def test__get_available(self, assessment_instance): - base_model = RepairModel_Base(assessment_instance) + def test__get_available(self, assessment_instance: Assessment) -> None: + base_model = RepairModel_DS(assessment_instance) base_model.loss_params = pd.DataFrame(index=['cmp.A', 'cmp.B', 'cmp.C']) - assert base_model._get_available() == {'cmp.A', 'cmp.B', 'cmp.C'} + assert base_model.get_available() == {'cmp.A', 'cmp.B', 'cmp.C'} class TestRepairModel_DS(TestRepairModel_Base): - def test__convert_loss_parameter_units(self, assessment_instance): + def test_convert_loss_parameter_units( + self, assessment_instance: Assessment + ) -> None: ds_model = RepairModel_DS(assessment_instance) ds_model.loss_params = pd.DataFrame( { @@ -627,7 +654,7 @@ def test__convert_loss_parameter_units(self, assessment_instance): index=pd.MultiIndex.from_tuples([('cmpA', 'Cost'), ('cmpB', 'Cost')]), ) - ds_model._convert_loss_parameter_units() + ds_model.convert_loss_parameter_units() # DVs are scaled by 3/2, quantities by 2 pd.testing.assert_frame_equal( @@ -646,7 +673,9 @@ def test__convert_loss_parameter_units(self, assessment_instance): ), ) - def test__drop_unused_damage_states(self, assessment_instance): + def test__drop_unused_damage_states( + self, assessment_instance: Assessment + ) -> None: ds_model = RepairModel_DS(assessment_instance) loss_params = pd.DataFrame( { @@ -660,16 +689,15 @@ def test__drop_unused_damage_states(self, assessment_instance): } ) ds_model.loss_params = loss_params - ds_model._drop_unused_damage_states() - pd.testing.assert_frame_equal(ds_model.loss_params, loss_params.iloc[0:4, :]) - - def test__create_DV_RVs(self, assessment_instance): + ds_model.drop_unused_damage_states() + pd.testing.assert_frame_equal(ds_model.loss_params, loss_params.iloc[:, 0:4]) + def test__create_DV_RVs(self, assessment_instance: Assessment) -> None: assessment_instance.options.rho_cost_time = 0.30 ds_model = RepairModel_DS(assessment_instance) ds_model.decision_variables = ('Cost', 'Time') - ds_model._missing = {('cmp.B', 'Cost'), ('cmp.B', 'Time')} - ds_model._loss_map = pd.DataFrame( + ds_model.missing = {('cmp.B', 'Cost'), ('cmp.B', 'Time')} + ds_model.loss_map = pd.DataFrame( { 'Repair': ['cmp.A', 'cmp.B', 'cmp.C', 'cmp.D', 'cmp.E'], }, @@ -708,6 +736,7 @@ def test__create_DV_RVs(self, assessment_instance): names=['cmp', 'loc', 'dir', 'uid', 'ds'], ) rv_reg = ds_model._create_DV_RVs(cases) + assert rv_reg is not None for key in ( 'Cost-cmp.A-1-0-1-0', 'Time-cmp.A-1-0-1-0', @@ -719,13 +748,13 @@ def test__create_DV_RVs(self, assessment_instance): assert isinstance(rv_reg.RV['Time-cmp.A-1-0-1-0'], uq.NormalRandomVariable) assert isinstance(rv_reg.RV['Cost-cmp.D-1-0-1-0'], uq.NormalRandomVariable) assert np.all( - rv_reg.RV['Cost-cmp.A-1-0-1-0'].theta[0:2] == np.array((1.0, 1.0)) + rv_reg.RV['Cost-cmp.A-1-0-1-0'].theta[0:2] == np.array((1.0, 1.0)) # type: ignore ) assert np.all( - rv_reg.RV['Time-cmp.A-1-0-1-0'].theta[0:2] == np.array((1.0, 1.0)) + rv_reg.RV['Time-cmp.A-1-0-1-0'].theta[0:2] == np.array((1.0, 1.0)) # type: ignore ) assert np.all( - rv_reg.RV['Cost-cmp.D-1-0-1-0'].theta[0:2] == np.array([1.0, 1.0]) + rv_reg.RV['Cost-cmp.D-1-0-1-0'].theta[0:2] == np.array([1.0, 1.0]) # type: ignore ) assert 'DV-cmp.A-1-0-1-0_set' in rv_reg.RV_set np.all( @@ -734,12 +763,13 @@ def test__create_DV_RVs(self, assessment_instance): ) assert len(rv_reg.RV_set) == 1 - def test__create_DV_RVs_all_deterministic(self, assessment_instance): - + def test__create_DV_RVs_all_deterministic( + self, assessment_instance: Assessment + ) -> None: ds_model = RepairModel_DS(assessment_instance) ds_model.decision_variables = ('myRV',) - ds_model._missing = set() - ds_model._loss_map = pd.DataFrame( + ds_model.missing = set() + ds_model.loss_map = pd.DataFrame( {'Repair': ['cmp.A']}, index=['cmp.A'], ) @@ -761,8 +791,9 @@ def test__create_DV_RVs_all_deterministic(self, assessment_instance): assert rv_reg is None - def test__calc_median_consequence_no_locs(self, assessment_instance): - + def test__calc_median_consequence_no_locs( + self, assessment_instance: Assessment + ) -> None: # Test the method when the eco_qnt dataframe's columns do not # contain `loc` information. @@ -782,7 +813,7 @@ def test__calc_median_consequence_no_locs(self, assessment_instance): # is_for_LF_model represents a component->consequence pair # that is intended for processing by the loss function model # and should be ignored by the damage state model. - ds_model._loss_map = pd.DataFrame( + ds_model.loss_map = pd.DataFrame( { 'Repair': ['cmp.A', 'cmp.B', 'missing_cmp', 'is_for_LF_model'], }, @@ -808,9 +839,10 @@ def test__calc_median_consequence_no_locs(self, assessment_instance): [('cmp.A', 'my_DV'), ('cmp.B', 'my_DV')] ), ).rename_axis(index=['Loss Driver', 'Decision Variable']) - ds_model._missing = {('missing_cmp', 'my_DV')} + ds_model.missing = {('missing_cmp', 'my_DV')} medians = ds_model._calc_median_consequence(eco_qnt) - assert len(medians) == 1 and 'my_DV' in medians + assert len(medians) == 1 + assert 'my_DV' in medians pd.testing.assert_frame_equal( medians['my_DV'], pd.DataFrame( @@ -838,14 +870,15 @@ def test__calc_median_consequence_no_locs(self, assessment_instance): }, index=pd.MultiIndex.from_tuples([('cmp.A', 'my_DV')]), ).rename_axis(index=['Loss Driver', 'Decision Variable']) - with pytest.raises(ValueError) as record: + with pytest.raises( + ValueError, + match='Loss Distribution of type multilinear_CDF not supported.', + ): ds_model._calc_median_consequence(eco_qnt) - assert 'Loss Distribution of type multilinear_CDF not supported.' in str( - record.value - ) - - def test__calc_median_consequence_locs(self, assessment_instance): + def test__calc_median_consequence_locs( + self, assessment_instance: Assessment + ) -> None: # Test the method when the eco_qnt dataframe's columns contain # `loc` information. @@ -862,7 +895,7 @@ def test__calc_median_consequence_locs(self, assessment_instance): # is_for_LF_model represents a component->consequence pair # that is intended for processing by the loss function model # and should be ignored by the damage state model. - ds_model._loss_map = pd.DataFrame( + ds_model.loss_map = pd.DataFrame( { 'Repair': ['cmp.A'], }, @@ -886,9 +919,10 @@ def test__calc_median_consequence_locs(self, assessment_instance): }, index=pd.MultiIndex.from_tuples([('cmp.A', 'my_DV')]), ).rename_axis(index=['Loss Driver', 'Decision Variable']) - ds_model._missing = set() + ds_model.missing = set() medians = ds_model._calc_median_consequence(eco_qnt) - assert len(medians) == 1 and 'my_DV' in medians + assert len(medians) == 1 + assert 'my_DV' in medians pd.testing.assert_frame_equal( medians['my_DV'], pd.DataFrame( @@ -900,8 +934,9 @@ def test__calc_median_consequence_locs(self, assessment_instance): class TestRepairModel_LF(TestRepairModel_Base): - - def test__convert_loss_parameter_units(self, assessment_instance): + def test_convert_loss_parameter_units( + self, assessment_instance: Assessment + ) -> None: lf_model = RepairModel_LF(assessment_instance) lf_model.loss_params = pd.DataFrame( { @@ -915,7 +950,7 @@ def test__convert_loss_parameter_units(self, assessment_instance): index=pd.MultiIndex.from_tuples([('cmpA', 'Cost'), ('cmpB', 'Cost')]), ) - lf_model._convert_loss_parameter_units() + lf_model.convert_loss_parameter_units() pd.testing.assert_frame_equal( lf_model.loss_params, @@ -934,8 +969,7 @@ def test__convert_loss_parameter_units(self, assessment_instance): ), ) - def test__calc_median_consequence(self, assessment_instance): - + def test__calc_median_consequence(self, assessment_instance: Assessment) -> None: lf_model = RepairModel_LF(assessment_instance) performance_group = pd.DataFrame( @@ -973,25 +1007,26 @@ def test__calc_median_consequence(self, assessment_instance): ) # test small interpolation domain warning demand_dict = {'PFA-1-1': np.array((1.00, 2.00, 1e3))} - with pytest.raises(ValueError) as record: + with pytest.raises( + ValueError, + match=re.escape( + 'Loss function interpolation for consequence ' + '`cmp.A-dv.A` has failed. Ensure a sufficient ' + 'interpolation domain for the X values ' + '(those after the `|` symbol) and verify ' + 'the X-value and Y-value lengths match.' + ), + ): lf_model._calc_median_consequence( performance_group, loss_map, required_edps, demand_dict, cmp_sample ) - assert ( - 'Loss function interpolation for consequence ' - '`cmp.A-dv.A` has failed. Ensure a sufficient ' - 'interpolation domain for the X values ' - '(those after the `|` symbol) and verify ' - 'the X-value and Y-value lengths match.' - ) in str(record.value) - - def test__create_DV_RVs(self, assessment_instance): + def test__create_DV_RVs(self, assessment_instance: Assessment) -> None: assessment_instance.options.rho_cost_time = 0.50 lf_model = RepairModel_LF(assessment_instance) lf_model.decision_variables = ('Cost', 'Time') - lf_model._missing = set() - lf_model._loss_map = pd.DataFrame( + lf_model.missing = set() + lf_model.loss_map = pd.DataFrame( { 'Repair': ['cmp.A', 'cmp.B'], }, @@ -1027,6 +1062,7 @@ def test__create_DV_RVs(self, assessment_instance): names=['dv', 'loss', 'dmg', 'loc', 'dir', 'uid', 'block'], ) rv_reg = lf_model._create_DV_RVs(cases) + assert rv_reg is not None for key in ( 'Cost-cmp.A-cmp.A-0-1-0-1', 'Time-cmp.A-cmp.A-0-1-0-1', @@ -1040,10 +1076,10 @@ def test__create_DV_RVs(self, assessment_instance): rv_reg.RV['Time-cmp.A-cmp.A-0-1-0-1'], uq.NormalRandomVariable ) assert np.all( - rv_reg.RV['Cost-cmp.A-cmp.A-0-1-0-1'].theta[0:2] == np.array((1.0, 0.3)) + rv_reg.RV['Cost-cmp.A-cmp.A-0-1-0-1'].theta[0:2] == np.array((1.0, 0.3)) # type: ignore ) assert np.all( - rv_reg.RV['Time-cmp.A-cmp.A-0-1-0-1'].theta[0:2] == np.array((1.0, 0.3)) + rv_reg.RV['Time-cmp.A-cmp.A-0-1-0-1'].theta[0:2] == np.array((1.0, 0.3)) # type: ignore ) assert 'DV-cmp.A-cmp.A-0-1-0-1_set' in rv_reg.RV_set np.all( @@ -1052,14 +1088,15 @@ def test__create_DV_RVs(self, assessment_instance): ) assert len(rv_reg.RV_set) == 1 - def test__create_DV_RVs_no_rv_case(self, assessment_instance): - + def test__create_DV_RVs_no_rv_case( + self, assessment_instance: Assessment + ) -> None: # Special case where there is no need for RVs lf_model = RepairModel_LF(assessment_instance) lf_model.decision_variables = ('Cost', 'Time') - lf_model._missing = set() - lf_model._loss_map = pd.DataFrame( + lf_model.missing = set() + lf_model.loss_map = pd.DataFrame( { 'Repair': ['cmp.B'], }, @@ -1090,16 +1127,16 @@ def test__create_DV_RVs_no_rv_case(self, assessment_instance): assert rv_reg is None -def test__prep_constant_median_DV(): +def test__prep_constant_median_DV() -> None: median = 10.00 - constant_median_DV = model.loss_model._prep_constant_median_DV(median) - assert constant_median_DV() == median + constant_median_dv = model.loss_model._prep_constant_median_DV(median) + assert constant_median_dv() == median values = (1.0, 2.0, 3.0, 4.0, 5.0) for value in values: - assert constant_median_DV(value) == 10.00 + assert constant_median_dv(value) == 10.00 -def test__prep_bounded_multilinear_median_DV(): +def test__prep_bounded_multilinear_median_DV() -> None: medians = np.array((1.00, 2.00, 3.00, 4.00, 5.00)) quantities = np.array((0.00, 1.00, 2.00, 3.00, 4.00)) f = model.loss_model._prep_bounded_multilinear_median_DV(medians, quantities) @@ -1128,12 +1165,18 @@ def test__prep_bounded_multilinear_median_DV(): expected_list = [3.5, 4.5] assert np.allclose(result_list, expected_list) - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match=( + 'A bounded linear median Decision Variable function ' + 'called without specifying the quantity ' + 'of damaged components' + ), + ): f(None) -def test__is_for_lf_model(): - +def test__is_for_lf_model() -> None: positive_case = pd.DataFrame( { ('LossFunction', 'Theta_0'): [0.5], @@ -1152,8 +1195,7 @@ def test__is_for_lf_model(): assert _is_for_lf_model(negative_case) is False -def test__is_for_ds_model(): - +def test__is_for_ds_model() -> None: positive_case = pd.DataFrame( { ('DS1', 'Theta_0'): [0.50], diff --git a/pelicun/tests/basic/test_model.py b/pelicun/tests/basic/test_model.py index 5c12de685..af878f089 100644 --- a/pelicun/tests/basic/test_model.py +++ b/pelicun/tests/basic/test_model.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -38,24 +37,22 @@ # Adam Zsarnóczay # John Vouvakis Manousakis -""" -This file defines a class used by the model unit tests. -""" +"""This file defines a class used by the model unit tests.""" from __future__ import annotations + from copy import deepcopy +from typing import Callable + import pytest -from pelicun import assessment -# pylint: disable=missing-function-docstring -# pylint: disable=missing-class-docstring -# pylint: disable=missing-return-doc,missing-return-type-doc +from pelicun import assessment class TestModelModule: @pytest.fixture - def assessment_factory(self): - def create_instance(verbose): + def assessment_factory(self) -> Callable: + def create_instance(*, verbose: bool) -> assessment.Assessment: x = assessment.Assessment() x.log.verbose = verbose return x @@ -63,5 +60,5 @@ def create_instance(verbose): return create_instance @pytest.fixture(params=[True, False]) - def assessment_instance(self, request, assessment_factory): - return deepcopy(assessment_factory(request.param)) + def assessment_instance(self, request, assessment_factory) -> None: # noqa: ANN001 + return deepcopy(assessment_factory(verbose=request.param)) diff --git a/pelicun/tests/basic/test_pelicun_model.py b/pelicun/tests/basic/test_pelicun_model.py index bc4541eef..6b37fd77d 100644 --- a/pelicun/tests/basic/test_pelicun_model.py +++ b/pelicun/tests/basic/test_pelicun_model.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -38,33 +37,34 @@ # Adam Zsarnóczay # John Vouvakis Manousakis -""" -These are unit and integration tests on the PelicunModel class. -""" +"""These are unit and integration tests on the PelicunModel class.""" from __future__ import annotations + from copy import deepcopy -import pytest +from typing import TYPE_CHECKING + import numpy as np import pandas as pd +import pytest + from pelicun import model from pelicun.tests.basic.test_model import TestModelModule - -# pylint: disable=missing-function-docstring -# pylint: disable=missing-class-docstring -# pylint: disable=missing-return-doc,missing-return-type-doc +if TYPE_CHECKING: + from pelicun.assessment import Assessment + from pelicun.model.pelicun_model import PelicunModel class TestPelicunModel(TestModelModule): @pytest.fixture - def pelicun_model(self, assessment_instance): + def pelicun_model(self, assessment_instance: Assessment) -> PelicunModel: return deepcopy(model.PelicunModel(assessment_instance)) - def test_init(self, pelicun_model): + def test_init(self, pelicun_model: PelicunModel) -> None: assert pelicun_model.log - def test__convert_marginal_params(self, pelicun_model): + def test__convert_marginal_params(self, pelicun_model: PelicunModel) -> None: # one row, only Theta_0, no conversion marginal_params = pd.DataFrame( [['1.0']], @@ -205,3 +205,13 @@ def test__convert_marginal_params(self, pelicun_model): pd.testing.assert_frame_equal( expected_df, res, check_index_type=False, check_column_type=False ) + + def test_query_error_setup(self, pelicun_model: PelicunModel) -> None: + assert ( + pelicun_model.query_error_setup( + 'Loss/ReplacementThreshold/RaiseOnUnknownKeys' + ) + is True + ) + with pytest.raises(KeyError): + pelicun_model.query_error_setup('some/invalid/path') diff --git a/pelicun/tests/basic/test_uq.py b/pelicun/tests/basic/test_uq.py index 7dfd1cceb..276e10367 100644 --- a/pelicun/tests/basic/test_uq.py +++ b/pelicun/tests/basic/test_uq.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -47,16 +46,22 @@ """ from __future__ import annotations + +import math +import re import warnings -import pytest + import numpy as np -from scipy.stats import norm # type: ignore -from scipy.stats import lognorm # type: ignore -from scipy.stats import weibull_min # type: ignore -from pelicun import uq -from pelicun.tests.util import import_pickle -from pelicun.tests.util import export_pickle +import pytest +from scipy.stats import ( + lognorm, # type: ignore + norm, # type: ignore + weibull_min, # type: ignore +) +from pelicun import uq +from pelicun.base import ensure_value +from pelicun.tests.util import export_pickle, import_pickle # The tests maintain the order of definitions of the `uq.py` file. @@ -69,54 +74,65 @@ # The following tests verify the functions of the module. -def test_scale_distribution(): +def test_scale_distribution() -> None: # used in all cases theta = np.array((-1.00, 1.00)) trunc = np.array((-2.00, 2.00)) # case 1: # normal distribution, factor of two - res = uq.scale_distribution(2.00, 'normal', theta, trunc) - assert np.allclose(res[0], np.array((-2.00, 1.00))) # theta_new - assert np.allclose(res[1], np.array((-4.00, 4.00))) # truncation_limits + theta_new, truncation_limits = uq.scale_distribution( + 2.00, 'normal', theta, trunc + ) + assert truncation_limits is not None + assert np.allclose(theta_new, np.array((-2.00, 1.00))) + assert np.allclose(truncation_limits, np.array((-4.00, 4.00))) # case 2: # normal_std distribution, factor of two res = uq.scale_distribution(2.00, 'normal_std', theta, trunc) + assert res[1] is not None assert np.allclose(res[0], np.array((-2.00, 2.00))) # theta_new assert np.allclose(res[1], np.array((-4.00, 4.00))) # truncation_limits # case 3: # normal_cov distribution, factor of two res = uq.scale_distribution(2.00, 'normal_cov', theta, trunc) + assert res[1] is not None assert np.allclose(res[0], np.array((-2.00, 1.00))) # theta_new assert np.allclose(res[1], np.array((-4.00, 4.00))) # truncation_limits # case 4: # lognormal distribution, factor of two res = uq.scale_distribution(2.00, 'lognormal', theta, trunc) + assert res[1] is not None assert np.allclose(res[0], np.array((-2.00, 1.00))) # theta_new assert np.allclose(res[1], np.array((-4.00, 4.00))) # truncation_limits # case 5: # uniform distribution, factor of two res = uq.scale_distribution(2.00, 'uniform', theta, trunc) + assert res[1] is not None assert np.allclose(res[0], np.array((-2.00, 2.00))) # theta_new assert np.allclose(res[1], np.array((-4.00, 4.00))) # truncation_limits # case 6: unsupported distribution - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match='Unsupported distribution: benktander-weibull' + ): uq.scale_distribution(0.50, 'benktander-weibull', np.array((1.00, 10.00))) -def test_mvn_orthotope_density(): +def test_mvn_orthotope_density() -> None: # case 1: # zero-width slice should result in a value of zero. mu_val = 0.00 cov_val = 1.00 lower_val = -1.00 upper_val = -1.00 - res = uq.mvn_orthotope_density(mu_val, cov_val, lower_val, upper_val) + res = uq.mvn_orthotope_density( + mu_val, np.atleast_2d([cov_val]), lower_val, upper_val + ) assert np.allclose(res, np.array((0.00, 2.00e-16))) # case 2: @@ -125,7 +141,9 @@ def test_mvn_orthotope_density(): cov_val = 1.00 lower_val = np.nan upper_val = 0.00 - res = uq.mvn_orthotope_density(mu_val, cov_val, lower_val, upper_val) + res = uq.mvn_orthotope_density( + mu_val, np.atleast_2d([cov_val]), lower_val, upper_val + ) assert np.allclose(res, np.array((0.50, 2.00e-16))) # case 3: @@ -134,7 +152,9 @@ def test_mvn_orthotope_density(): cov_val = 1.00 lower_val = 0.00 upper_val = np.nan - res = uq.mvn_orthotope_density(mu_val, cov_val, lower_val, upper_val) + res = uq.mvn_orthotope_density( + mu_val, np.atleast_2d([cov_val]), lower_val, upper_val + ) assert np.allclose(res, np.array((0.50, 2.00e-16))) # case 4: @@ -167,13 +187,12 @@ def test_mvn_orthotope_density(): assert np.allclose(res, np.array((1.00 / 8.00, 2.00e-16))) -def test__get_theta(): - +def test__get_theta() -> None: # Evaluate uq._get_theta() for some valid inputs res = uq._get_theta( np.array(((1.00, 1.00), (1.00, 0.5), (0.00, 0.3), (1.50, 0.2))), np.array(((0.00, 1.00), (1.00, 0.5), (0.00, 0.3), (1.00, 0.2))), - ['normal', 'lognormal', 'normal_std', 'normal_cov'], + np.array(['normal', 'lognormal', 'normal_std', 'normal_cov']), ) # Check that the expected output is obtained for each distribution type @@ -184,11 +203,15 @@ def test__get_theta(): assert np.allclose(res, expected_res) # Check that it fails for invalid inputs - with pytest.raises(ValueError): - uq._get_theta(np.array((1.00,)), np.array((1.00,)), 'not_a_distribution') + with pytest.raises( + ValueError, match='Unsupported distribution: not_a_distribution' + ): + uq._get_theta( + np.array((1.00,)), np.array((1.00,)), np.array(['not_a_distribution']) + ) -def test__get_limit_probs(): +def test__get_limit_probs() -> None: # verify that it works for valid inputs res = uq._get_limit_probs( @@ -233,7 +256,9 @@ def test__get_limit_probs(): # verify that it fails for invalid inputs - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match='Unsupported distribution: not_a_distribution' + ): uq._get_limit_probs( np.array((1.00,)), 'not_a_distribution', @@ -241,7 +266,7 @@ def test__get_limit_probs(): ) -def test__get_std_samples(): +def test__get_std_samples() -> None: # test that it works with valid inputs # case 1: @@ -251,7 +276,7 @@ def test__get_std_samples(): tr_limits = np.array(((np.nan, np.nan),)) dist_list = np.array(('normal',)) res = uq._get_std_samples(samples, theta, tr_limits, dist_list) - assert np.allclose(res, np.array(((1.00, 2.00, 3.00)))) + assert np.allclose(res, np.array((1.00, 2.00, 3.00))) # case 2: # multivariate samples @@ -290,8 +315,9 @@ def test__get_std_samples(): ) # test that it fails for invalid inputs - - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match='Unsupported distribution: some_unsupported_distribution' + ): uq._get_std_samples( np.array(((1.00, 2.00, 3.00),)), np.array(((0.00, 1.0),)), @@ -300,43 +326,50 @@ def test__get_std_samples(): ) -def test__get_std_corr_matrix(): +def test__get_std_corr_matrix() -> None: # test that it works with valid inputs # case 1: std_samples = np.array(((1.00,),)) res = uq._get_std_corr_matrix(std_samples) + assert res is not None assert np.allclose(res, np.array(((1.00,),))) # case 2: std_samples = np.array(((1.00, 0.00), (0.00, 1.00))) res = uq._get_std_corr_matrix(std_samples) + assert res is not None assert np.allclose(res, np.array(((1.00, 0.00), (0.00, 1.00)))) # case 3: std_samples = np.array(((1.00, 0.00), (0.00, -1.00))) res = uq._get_std_corr_matrix(std_samples) + assert res is not None assert np.allclose(res, np.array(((1.00, 0.00), (0.00, 1.00)))) # case 4: std_samples = np.array(((1.00, 1.00), (1.00, 1.00))) res = uq._get_std_corr_matrix(std_samples) + assert res is not None assert np.allclose(res, np.array(((1.00, 1.00), (1.00, 1.00)))) # case 5: std_samples = np.array(((1.00, 1e50), (-1.00, -1.00))) res = uq._get_std_corr_matrix(std_samples) + assert res is not None assert np.allclose(res, np.array(((1.00, 0.00), (0.00, 1.00)))) # test that it fails for invalid inputs for bad_item in (np.nan, np.inf, -np.inf): - with pytest.raises(ValueError): - x = np.array(((1.00, bad_item), (-1.00, -1.00))) + x = np.array(((1.00, bad_item), (-1.00, -1.00))) + with pytest.raises( + ValueError, match='std_samples array must not contain inf or NaN values' + ): uq._get_std_corr_matrix(x) -def test__mvn_scale(): +def test__mvn_scale() -> None: # case 1: np.random.seed(40) sample = np.random.normal(0.00, 1.00, size=(2, 5)).T @@ -352,7 +385,7 @@ def test__mvn_scale(): assert np.allclose(res, np.array((0.0, 0.0, 0.0, 0.0, 0.0))) -def test__neg_log_likelihood(): +def test__neg_log_likelihood() -> None: # Parameters not within the pre-defined bounds should yield a # large value to discourage the optimization algorithm from going # in that direction. @@ -367,9 +400,9 @@ def test__neg_log_likelihood(): (1.10, 0.30), ), ), - dist_list=['normal', 'normal'], - tr_limits=[None, None], - det_limits=[None, None], + dist_list=np.array(('normal', 'normal')), + tr_limits=np.array((np.nan, np.nan)), + det_limits=[np.array((np.nan, np.nan))], censored_count=0, enforce_bounds=True, ) @@ -380,17 +413,17 @@ def test__neg_log_likelihood(): res = uq._neg_log_likelihood( np.array((np.nan, 0.20)), np.array((1.00, 0.20)), - 0.00, - 20.00, + np.atleast_1d((0.00,)), + np.atleast_1d((20.00,)), np.array( ( (0.90, 0.10), (1.10, 0.30), ), ), - ['normal', 'normal'], - [-np.inf, np.inf], - [np.nan, np.nan], + np.array(('normal', 'normal')), + np.array((-np.inf, np.inf)), + [np.array((np.nan, np.nan))], 0, enforce_bounds=False, ) @@ -398,7 +431,7 @@ def test__neg_log_likelihood(): assert res == 1e10 -def test_fit_distribution_to_sample_univariate(): +def test_fit_distribution_to_sample_univariate() -> None: # a single value in the sample sample_vec = np.array((1.00,)) res = uq.fit_distribution_to_sample(sample_vec, 'normal') @@ -481,10 +514,10 @@ def test_fit_distribution_to_sample_univariate(): usable_sample, 'normal_cov', censored_count=c_count, - detection_limits=[c_lower, c_upper], + detection_limits=(c_lower, c_upper), ) compare_a = ( - np.array(((1.13825975, 0.46686491))), + np.array((1.13825975, 0.46686491)), np.array( ((1.00,)), ), @@ -496,10 +529,10 @@ def test_fit_distribution_to_sample_univariate(): usable_sample, 'normal_std', censored_count=c_count, - detection_limits=[c_lower, c_upper], + detection_limits=(c_lower, c_upper), ) compare_a = ( - np.array(((1.13825975, 0.53141375))), + np.array((1.13825975, 0.53141375)), np.array( ((1.00,)), ), @@ -520,10 +553,10 @@ def test_fit_distribution_to_sample_univariate(): usable_sample, 'normal_cov', censored_count=c_count, - detection_limits=[c_lower, c_upper], + detection_limits=(c_lower, c_upper), ) compare_b = ( - np.array(((-1.68598848, 1.75096914))), + np.array((-1.68598848, 1.75096914)), np.array( ((1.00,)), ), @@ -544,10 +577,10 @@ def test_fit_distribution_to_sample_univariate(): usable_sample, 'normal_cov', censored_count=c_count, - detection_limits=[c_lower, c_upper], + detection_limits=(c_lower, c_upper), ) compare_c = ( - np.array(((1.68598845, 1.75096921))), + np.array((1.68598845, 1.75096921)), np.array( ((1.00,)), ), @@ -565,9 +598,12 @@ def test_fit_distribution_to_sample_univariate(): sample_vec = np.array((-3.00, -2.00, -1.00, 0.00, 1.00, 2.00, 3.00)).reshape( (1, -1) ) - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match='One or more sample values lie outside of the specified truncation limits.', + ): res = uq.fit_distribution_to_sample( - sample_vec, 'normal_cov', truncation_limits=[t_lower, t_upper] + sample_vec, 'normal_cov', truncation_limits=(t_lower, t_upper) ) # truncated data, only lower, expect failure @@ -576,9 +612,15 @@ def test_fit_distribution_to_sample_univariate(): sample_vec = np.array((-3.00, -2.00, -1.00, 0.00, 1.00, 2.00, 3.00)).reshape( (1, -1) ) - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match=( + 'One or more sample values lie ' + 'outside of the specified truncation limits.' + ), + ): res = uq.fit_distribution_to_sample( - sample_vec, 'normal_cov', truncation_limits=[t_lower, t_upper] + sample_vec, 'normal_cov', truncation_limits=(t_lower, t_upper) ) # truncated data, only upper, expect failure @@ -587,9 +629,15 @@ def test_fit_distribution_to_sample_univariate(): sample_vec = np.array((-3.00, -2.00, -1.00, 0.00, 1.00, 2.00, 3.00)).reshape( (1, -1) ) - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match=( + 'One or more sample values lie ' + 'outside of the specified truncation limits.' + ), + ): res = uq.fit_distribution_to_sample( - sample_vec, 'normal_cov', truncation_limits=[t_lower, t_upper] + sample_vec, 'normal_cov', truncation_limits=(t_lower, t_upper) ) # truncated data, lower and upper @@ -598,10 +646,10 @@ def test_fit_distribution_to_sample_univariate(): t_upper = +4.50 sample_vec = np.array((0.00, 1.00, 2.00, 3.00, 4.00)).reshape((1, -1)) res_a = uq.fit_distribution_to_sample( - sample_vec, 'normal_cov', truncation_limits=[t_lower, t_upper] + sample_vec, 'normal_cov', truncation_limits=(t_lower, t_upper) ) compare_a = ( - np.array(((1.99999973, 2.2639968))), + np.array((1.99999973, 2.2639968)), np.array( ((1.00,)), ), @@ -617,9 +665,9 @@ def test_fit_distribution_to_sample_univariate(): (1, -1) ) res_b = uq.fit_distribution_to_sample( - sample_vec, 'normal_cov', truncation_limits=[t_lower, t_upper] + sample_vec, 'normal_cov', truncation_limits=(t_lower, t_upper) ) - compare_b = (np.array(((-0.09587816, 21.95601487))), np.array(((1.00,)))) + compare_b = (np.array((-0.09587816, 21.95601487)), np.array((1.00,))) assert np.allclose(res_b[0], compare_b[0]) assert np.allclose(res_b[1], compare_b[1]) @@ -631,10 +679,10 @@ def test_fit_distribution_to_sample_univariate(): (1, -1) ) res_c = uq.fit_distribution_to_sample( - sample_vec, 'normal_cov', truncation_limits=[t_lower, t_upper] + sample_vec, 'normal_cov', truncation_limits=(t_lower, t_upper) ) compare_c = ( - np.array(((0.09587811, 21.95602574))), + np.array((0.09587811, 21.95602574)), np.array( ((1.00,)), ), @@ -647,7 +695,7 @@ def test_fit_distribution_to_sample_univariate(): assert np.isclose(res_b[0][0, 1], res_c[0][0, 1]) -def test_fit_distribution_to_sample_multivariate(): +def test_fit_distribution_to_sample_multivariate() -> None: # uncorrelated, normal np.random.seed(40) sample = np.random.multivariate_normal( @@ -687,8 +735,8 @@ def test_fit_distribution_to_sample_multivariate(): res = uq.fit_distribution_to_sample( sample, ['normal_cov', 'normal_cov'], - truncation_limits=np.array((-5.00, 6.00)), - detection_limits=np.array((0.20, 1.80)), + truncation_limits=(-5.00, 6.00), + detection_limits=(0.20, 1.80), ) compare = ( np.array(((1.00833201, 1.0012552), (1.00828936, 0.99477853))), @@ -701,12 +749,12 @@ def test_fit_distribution_to_sample_multivariate(): np.random.seed(40) sample = np.full( (2, 10), - 3.14, + 123.00, ) np.random.seed(40) res = uq.fit_distribution_to_sample(sample, ['normal_cov', 'normal_cov']) compare = ( - np.array(((3.14, 1.0e-6), (3.14, 1.0e-6))), + np.array(((123.00, 1.0e-6), (123.00, 1.0e-6))), np.array(((1.00, 0.00), (0.00, 1.00))), ) assert np.allclose(res[0], compare[0]) @@ -721,7 +769,7 @@ def test_fit_distribution_to_sample_multivariate(): ) np.random.seed(40) res = uq.fit_distribution_to_sample( - sample, ['lognormal', 'lognormal'], detection_limits=np.array((1e-8, 5.00)) + sample, ['lognormal', 'lognormal'], detection_limits=(1e-8, 5.00) ) compare = ( np.array(((4.60517598e00, 2.18581908e-04), (4.60517592e00, 2.16575944e-04))), @@ -735,7 +783,7 @@ def test_fit_distribution_to_sample_multivariate(): np.random.seed(40) sample = np.full( (1, 10), - 3.14, + math.pi, ) np.random.seed(40) with pytest.raises(IndexError): @@ -780,14 +828,16 @@ def test_fit_distribution_to_sample_multivariate(): sample = np.concatenate( (np.random.normal(0.00, 1.00, size=100000), np.array((np.inf,))) ) - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match='Conversion to standard normal space was unsuccessful' + ): uq.fit_distribution_to_sample(sample, ['normal_cov']) -def test_fit_distribution_to_percentiles(): +def test_fit_distribution_to_percentiles() -> None: # normal, mean of 20 and standard deviation of 10 - percentiles = np.linspace(0.01, 0.99, num=10000) - values = norm.ppf(percentiles, loc=20, scale=10) + percentiles = np.linspace(0.01, 0.99, num=10000).tolist() + values = norm.ppf(percentiles, loc=20, scale=10).tolist() res = uq.fit_distribution_to_percentiles( values, percentiles, ['normal', 'lognormal'] ) @@ -795,7 +845,7 @@ def test_fit_distribution_to_percentiles(): assert np.allclose(res[1], np.array((20.00, 10.00))) # lognormal, median of 20 and beta of 0.4 - ln_values = lognorm.ppf(percentiles, s=0.40, scale=20.00) + ln_values = lognorm.ppf(percentiles, s=0.40, scale=20.00).tolist() res = uq.fit_distribution_to_percentiles( ln_values, percentiles, ['normal', 'lognormal'] ) @@ -803,17 +853,19 @@ def test_fit_distribution_to_percentiles(): assert np.allclose(res[1], np.array((20.0, 0.40))) # unrecognized distribution family - percentiles = np.linspace(0.01, 0.99, num=10000) - values = norm.ppf(percentiles, loc=20, scale=10) - with pytest.raises(ValueError): + percentiles = np.linspace(0.01, 0.99, num=10000).tolist() + values = norm.ppf(percentiles, loc=20, scale=10).tolist() + with pytest.raises( + ValueError, match='Distribution family not recognized: birnbaum-saunders' + ): uq.fit_distribution_to_percentiles( values, percentiles, ['lognormal', 'birnbaum-saunders'] ) -def test__OLS_percentiles(): +def test__OLS_percentiles() -> None: # normal: negative standard deviation - params = np.array((2.50, -0.10)) + params = (2.50, -0.10) perc = np.linspace(1e-2, 1.00 - 1e-2, num=5) values = norm.ppf(perc, loc=20, scale=10) family = 'normal' @@ -821,7 +873,7 @@ def test__OLS_percentiles(): assert res == 10000000000.0 # lognormal: negative median - params = np.array((-1.00, 0.40)) + params = (-1.00, 0.40) perc = np.linspace(1e-2, 1.00 - 1e-2, num=5) values = lognorm.ppf(perc, s=0.40, scale=20.00) family = 'lognormal' @@ -838,64 +890,67 @@ def test__OLS_percentiles(): # The following tests verify the methods of the objects of the module. -def test_NormalRandomVariable(): +def test_NormalRandomVariable() -> None: rv = uq.NormalRandomVariable('rv_name', theta=np.array((0.00, 1.00))) assert rv.name == 'rv_name' + assert rv.theta is not None np.testing.assert_allclose(rv.theta, np.array((0.00, 1.00))) assert np.all(np.isnan(rv.truncation_limits)) assert rv.RV_set is None assert rv.sample_DF is None # confirm that creating an attribute on the fly is not allowed with pytest.raises(AttributeError): - # pylint: disable=assigning-non-slot - # thanks pylint, we are aware of this. - rv.xyz = 123 + rv.xyz = 123 # type: ignore -def test_Normal_STD(): +def test_Normal_STD() -> None: rv = uq.Normal_STD('rv_name', theta=np.array((0.00, 1.00))) assert rv.name == 'rv_name' + assert rv.theta is not None np.testing.assert_allclose(rv.theta, np.array((0.00, 1.00))) assert np.all(np.isnan(rv.truncation_limits)) assert rv.RV_set is None assert rv.sample_DF is None with pytest.raises(AttributeError): - rv.xyz = 123 + rv.xyz = 123 # type: ignore -def test_Normal_COV(): - with pytest.raises(ValueError): +def test_Normal_COV() -> None: + with pytest.raises( + ValueError, match='The mean of Normal_COV RVs cannot be zero.' + ): rv = uq.Normal_COV('rv_name', theta=np.array((0.00, 1.00))) rv = uq.Normal_COV('rv_name', theta=np.array((2.00, 1.00))) assert rv.name == 'rv_name' + assert rv.theta is not None np.testing.assert_allclose(rv.theta, np.array((2.00, 2.00))) assert np.all(np.isnan(rv.truncation_limits)) assert rv.RV_set is None assert rv.sample_DF is None with pytest.raises(AttributeError): - rv.xyz = 123 + rv.xyz = 123 # type: ignore -def test_NormalRandomVariable_cdf(): +def test_NormalRandomVariable_cdf() -> None: # test CDF method rv = uq.NormalRandomVariable( 'test_rv', - theta=(1.0, 1.0), + theta=np.array((1.0, 1.0)), truncation_limits=np.array((0.00, np.nan)), ) # evaluate CDF at different points - x = (-1.0, 0.0, 0.5, 1.0, 2.0) + x = np.array((-1.0, 0.0, 0.5, 1.0, 2.0)) cdf = rv.cdf(x) # assert that CDF values are correct assert np.allclose(cdf, (0.0, 0.0, 0.1781461, 0.40571329, 0.81142658), rtol=1e-5) # repeat without truncation limits - rv = uq.NormalRandomVariable('test_rv', theta=(1.0, 1.0)) + rv = uq.NormalRandomVariable('test_rv', theta=np.array((1.0, 1.0))) # evaluate CDF at different points - x = (-1.0, 0.0, 0.5, 1.0, 2.0) + x = np.array((-1.0, 0.0, 0.5, 1.0, 2.0)) cdf = rv.cdf(x) # assert that CDF values are correct @@ -904,73 +959,110 @@ def test_NormalRandomVariable_cdf(): ) -def test_Normal_STD_cdf(): +def test_NormalRandomVariable_variable_theta_cdf() -> None: + rv = uq.NormalRandomVariable( + 'test_rv', + theta=np.array(((0.0, 1.0), (1.0, 1.0), (2.0, 1.0))), + truncation_limits=np.array((0.00, np.nan)), + ) + + # evaluate CDF at different points + x = np.array((-1.0, 0.5, 2.0)) + cdf = rv.cdf(x) + + # assert that CDF values are correct + expected_cdf = (0.0, 0.1781461, 0.48836013) + + assert np.allclose(cdf, expected_cdf, rtol=1e-5) + + # repeat without truncation limits + rv = uq.NormalRandomVariable( + 'test_rv', + theta=np.array(((0.0, 1.0), (1.0, 1.0), (2.0, 1.0))), + ) + + x = np.array((-1.0, 0.5, 2.0)) + cdf = rv.cdf(x) + + # assert that CDF values are correct + expected_cdf = (0.15865525, 0.30853754, 0.5) + assert np.allclose(cdf, expected_cdf, rtol=1e-5) + + +def test_Normal_STD_cdf() -> None: rv = uq.Normal_STD( 'test_rv', - theta=(1.0, 1.0), + theta=np.array((1.0, 1.0)), truncation_limits=np.array((0.00, np.nan)), ) - x = (-1.0, 0.0, 0.5, 1.0, 2.0) + x = np.array((-1.0, 0.0, 0.5, 1.0, 2.0)) cdf = rv.cdf(x) assert np.allclose(cdf, (0.0, 0.0, 0.1781461, 0.40571329, 0.81142658), rtol=1e-5) -def test_Normal_COV_cdf(): +def test_Normal_COV_cdf() -> None: rv = uq.Normal_COV( 'test_rv', - theta=(1.0, 1.0), + theta=np.array((1.0, 1.0)), truncation_limits=np.array((0.00, np.nan)), ) - x = (-1.0, 0.0, 0.5, 1.0, 2.0) + x = np.array((-1.0, 0.0, 0.5, 1.0, 2.0)) cdf = rv.cdf(x) assert np.allclose(cdf, (0.0, 0.0, 0.1781461, 0.40571329, 0.81142658), rtol=1e-5) -def test_NormalRandomVariable_inverse_transform(): +def test_NormalRandomVariable_inverse_transform() -> None: samples = np.array((0.10, 0.20, 0.30)) - rv = uq.NormalRandomVariable('test_rv', theta=(1.0, 0.5)) + rv = uq.NormalRandomVariable('test_rv', theta=np.array((1.0, 0.5))) rv.uni_sample = samples rv.inverse_transform_sampling() inverse_transform = rv.sample + assert inverse_transform is not None assert np.allclose( inverse_transform, np.array((0.35922422, 0.57918938, 0.73779974)), rtol=1e-5 ) - rv = uq.NormalRandomVariable('test_rv', theta=(1.0, 0.5)) - with pytest.raises(ValueError): + rv = uq.NormalRandomVariable('test_rv', theta=np.array((1.0, 0.5))) + with pytest.raises(ValueError, match='No available uniform sample.'): rv.inverse_transform_sampling() # with truncation limits rv = uq.NormalRandomVariable( - 'test_rv', theta=(1.0, 0.5), truncation_limits=(np.nan, 1.20) + 'test_rv', + theta=np.array((1.0, 0.5)), + truncation_limits=np.array((np.nan, 1.20)), ) rv.uni_sample = samples rv.inverse_transform_sampling() - inverse_transform = rv.sample + inverse_transform = ensure_value(rv.sample) assert np.allclose( inverse_transform, np.array((0.24508018, 0.43936, 0.57313359)), rtol=1e-5 ) rv = uq.NormalRandomVariable( - 'test_rv', theta=(1.0, 0.5), truncation_limits=(0.80, np.nan) + 'test_rv', + theta=np.array((1.0, 0.5)), + truncation_limits=np.array((0.80, np.nan)), ) rv.uni_sample = samples rv.inverse_transform_sampling() - inverse_transform = rv.sample + inverse_transform = ensure_value(rv.sample) assert np.allclose( inverse_transform, np.array((0.8863824, 0.96947866, 1.0517347)), rtol=1e-5 ) rv = uq.NormalRandomVariable( - 'test_rv', theta=(1.0, 0.5), truncation_limits=(0.80, 1.20) + 'test_rv', + theta=np.array((1.0, 0.5)), + truncation_limits=np.array((0.80, 1.20)), ) rv.uni_sample = samples rv.inverse_transform_sampling() - inverse_transform = rv.sample + inverse_transform = ensure_value(rv.sample) assert np.allclose( inverse_transform, np.array((0.84155378, 0.88203946, 0.92176503)), rtol=1e-5 ) @@ -981,47 +1073,81 @@ def test_NormalRandomVariable_inverse_transform(): # normal with problematic truncation limits rv = uq.NormalRandomVariable( - 'test_rv', theta=(1.0, 0.5), truncation_limits=(1e8, 2e8) + 'test_rv', theta=np.array((1.0, 0.5)), truncation_limits=np.array((1e8, 2e8)) ) rv.uni_sample = samples - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match=( + 'The probability mass within the truncation ' + 'limits is too small and the truncated ' + 'distribution cannot be sampled with ' + 'sufficiently high accuracy. This is most probably ' + 'due to incorrect truncation limits set ' + 'for the distribution.' + ), + ): rv.inverse_transform_sampling() -def test_Normal_STD_inverse_transform(): +def test_NormalRandomVariable_variable_theta_inverse_transform() -> None: + rv = uq.NormalRandomVariable( + 'test_rv', + theta=np.array( + ( + (0.0, 1.0), + (1.0, 1.0), + (2.0, 1.0), + (1.0, 1.0), + (1.0, 1.0), + (1.0, 1.0), + (1.0, 1.0), + ) + ), + ) + rv.uni_sample = np.array((0.5, 0.5, 0.5, 0.0, 1.0, 0.20, 0.80)) + rv.inverse_transform_sampling() + inverse_transform = rv.sample + expected_result = np.array( + (0.0, 1.0, 2.0, -np.inf, np.inf, 0.15837877, 1.84162123) + ) + assert inverse_transform is not None + assert np.allclose(inverse_transform, expected_result, rtol=1e-5) + + +def test_Normal_STD_inverse_transform() -> None: samples = np.array((0.10, 0.20, 0.30)) - rv = uq.Normal_STD('test_rv', theta=(1.0, 0.5)) + rv = uq.Normal_STD('test_rv', theta=np.array((1.0, 0.5))) rv.uni_sample = samples rv.inverse_transform_sampling() - inverse_transform = rv.sample + inverse_transform = ensure_value(rv.sample) assert np.allclose( inverse_transform, np.array((0.35922422, 0.57918938, 0.73779974)), rtol=1e-5 ) -def test_Normal_COV_inverse_transform(): +def test_Normal_COV_inverse_transform() -> None: samples = np.array((0.10, 0.20, 0.30)) - rv = uq.Normal_COV('test_rv', theta=(1.0, 0.5)) + rv = uq.Normal_COV('test_rv', theta=np.array((1.0, 0.5))) rv.uni_sample = samples rv.inverse_transform_sampling() - inverse_transform = rv.sample + inverse_transform = ensure_value(rv.sample) assert np.allclose( inverse_transform, np.array((0.35922422, 0.57918938, 0.73779974)), rtol=1e-5 ) -def test_LogNormalRandomVariable_cdf(): +def test_LogNormalRandomVariable_cdf() -> None: # lower truncation rv = uq.LogNormalRandomVariable( 'test_rv', - theta=(1.0, 1.0), + theta=np.array((1.0, 1.0)), truncation_limits=np.array((0.10, np.nan)), ) # confirm that creating an attribute on the fly is not allowed with pytest.raises(AttributeError): - # pylint: disable=assigning-non-slot - rv.xyz = 123 - x = (-1.0, 0.0, 0.5, 1.0, 2.0) + rv.xyz = 123 # type: ignore + x = np.array((-1.0, 0.0, 0.5, 1.0, 2.0)) cdf = rv.cdf(x) assert np.allclose( cdf, (0.0, 0.0, 0.23597085, 0.49461712, 0.75326339), rtol=1e-5 @@ -1030,29 +1156,59 @@ def test_LogNormalRandomVariable_cdf(): # upper truncation rv = uq.LogNormalRandomVariable( 'test_rv', - theta=(1.0, 1.0), + theta=np.array((1.0, 1.0)), truncation_limits=np.array((np.nan, 5.00)), ) - x = (-1.0, 0.0, 0.5, 1.0, 2.0) + x = np.array((-1.0, 0.0, 0.5, 1.0, 2.0)) cdf = rv.cdf(x) assert np.allclose( cdf, (0.00, 0.00, 0.25797755, 0.52840734, 0.79883714), rtol=1e-5 ) # no truncation - rv = uq.LogNormalRandomVariable('test_rv', theta=(1.0, 1.0)) - x = (-1.0, 0.0, 0.5, 1.0, 2.0) + rv = uq.LogNormalRandomVariable('test_rv', theta=np.array((1.0, 1.0))) + x = np.array((-1.0, 0.0, 0.5, 1.0, 2.0)) cdf = rv.cdf(x) assert np.allclose(cdf, (0.0, 0.0, 0.2441086, 0.5, 0.7558914), rtol=1e-5) -def test_LogNormalRandomVariable_inverse_transform(): +def test_LogNormalRandomVariable_variable_theta_cdf() -> None: + rv = uq.LogNormalRandomVariable( + 'test_rv', + theta=np.array(((0.1, 1.0), (1.0, 1.0), (2.0, 1.0))), + truncation_limits=np.array((0.20, 3.0)), + ) + + # evaluate CDF at different points + x = np.array((0.01, 0.5, 2.0)) + cdf = rv.cdf(x) + + # assert that CDF values are correct + expected_cdf = (0.0, 0.23491926, 0.75659125) + + assert np.allclose(cdf, expected_cdf, rtol=1e-5) + + # repeat without truncation limits + rv = uq.LogNormalRandomVariable( + 'test_rv', + theta=np.array(((0.01, 1.0), (1.0, 1.0), (2.0, 1.0))), + ) + + x = np.array((-1.0, 0.5, 2.0)) + cdf = rv.cdf(x) + + # assert that CDF values are correct + expected_cdf = (0.0, 0.2441086, 0.5) + assert np.allclose(cdf, expected_cdf, rtol=1e-5) + + +def test_LogNormalRandomVariable_inverse_transform() -> None: samples = np.array((0.10, 0.20, 0.30)) - rv = uq.LogNormalRandomVariable('test_rv', theta=(1.0, 0.5)) + rv = uq.LogNormalRandomVariable('test_rv', theta=np.array((1.0, 0.5))) rv.uni_sample = samples rv.inverse_transform_sampling() - inverse_transform = rv.sample + inverse_transform = ensure_value(rv.sample) assert np.allclose( inverse_transform, np.array((0.52688352, 0.65651442, 0.76935694)), rtol=1e-5 @@ -1064,12 +1220,12 @@ def test_LogNormalRandomVariable_inverse_transform(): rv = uq.LogNormalRandomVariable( 'test_rv', - theta=(1.0, 0.5), + theta=np.array((1.0, 0.5)), truncation_limits=np.array((0.50, np.nan)), ) rv.uni_sample = samples rv.inverse_transform_sampling() - inverse_transform = rv.sample + inverse_transform = ensure_value(rv.sample) assert np.allclose( inverse_transform, np.array((0.62614292, 0.73192471, 0.83365823)), rtol=1e-5 ) @@ -1079,85 +1235,107 @@ def test_LogNormalRandomVariable_inverse_transform(): # # lognormal without values to sample from - rv = uq.LogNormalRandomVariable('test_rv', theta=(1.0, 0.5)) - with pytest.raises(ValueError): + rv = uq.LogNormalRandomVariable('test_rv', theta=np.array((1.0, 0.5))) + with pytest.raises(ValueError, match='No available uniform sample.'): rv.inverse_transform_sampling() -def test_UniformRandomVariable_cdf(): +def test_LogNormalRandomVariable_variable_theta_inverse_transform() -> None: + rv = uq.LogNormalRandomVariable( + 'test_rv', + theta=np.array( + ( + (0.10, 1.0), + (1.0, 1.0), + (2.0, 1.0), + (1.0, 1.0), + (1.0, 1.0), + (1.0, 1.0), + (1.0, 1.0), + ) + ), + ) + rv.uni_sample = np.array((0.5, 0.5, 0.5, 0.0, 1.0, 0.20, 0.80)) + rv.inverse_transform_sampling() + inverse_transform = rv.sample + expected_result = np.array((0.1, 1.0, 2.0, 0.0, np.inf, 0.43101119, 2.32012539)) + assert inverse_transform is not None + assert np.allclose(inverse_transform, expected_result, rtol=1e-5) + + +def test_UniformRandomVariable_cdf() -> None: # uniform, both theta values - rv = uq.UniformRandomVariable('test_rv', theta=(0.0, 1.0)) + rv = uq.UniformRandomVariable('test_rv', theta=np.array((0.0, 1.0))) # confirm that creating an attribute on the fly is not allowed with pytest.raises(AttributeError): - # pylint: disable=assigning-non-slot - rv.xyz = 123 - x = (-1.0, 0.0, 0.5, 1.0, 2.0) + rv.xyz = 123 # type: ignore + x = np.array((-1.0, 0.0, 0.5, 1.0, 2.0)) cdf = rv.cdf(x) assert np.allclose(cdf, (0.0, 0.0, 0.5, 1.0, 1.0), rtol=1e-5) with warnings.catch_warnings(): warnings.simplefilter('ignore') # uniform, only upper theta value ( -inf implied ) - rv = uq.UniformRandomVariable('test_rv', theta=(np.nan, 100.00)) - x = (-1.0, 0.0, 0.5, 1.0, 2.0) + rv = uq.UniformRandomVariable('test_rv', theta=np.array((np.nan, 100.00))) + x = np.array((-1.0, 0.0, 0.5, 1.0, 2.0)) cdf = rv.cdf(x) assert np.all(np.isnan(cdf)) # uniform, only lower theta value ( +inf implied ) - rv = uq.UniformRandomVariable('test_rv', theta=(0.00, np.nan)) - x = (-1.0, 0.0, 0.5, 1.0, 2.0) + rv = uq.UniformRandomVariable('test_rv', theta=np.array((0.00, np.nan))) + x = np.array((-1.0, 0.0, 0.5, 1.0, 2.0)) cdf = rv.cdf(x) assert np.allclose(cdf, (0.0, 0.0, 0.0, 0.0, 0.0), rtol=1e-5) # uniform, with truncation limits rv = uq.UniformRandomVariable( 'test_rv', - theta=(0.0, 10.0), + theta=np.array((0.0, 10.0)), truncation_limits=np.array((0.00, 1.00)), ) - x = (-1.0, 0.0, 0.5, 1.0, 2.0) + x = np.array((-1.0, 0.0, 0.5, 1.0, 2.0)) cdf = rv.cdf(x) assert np.allclose(cdf, (0.0, 0.0, 0.5, 1.0, 1.0), rtol=1e-5) -def test_UniformRandomVariable_inverse_transform(): - rv = uq.UniformRandomVariable('test_rv', theta=(0.0, 1.0)) +def test_UniformRandomVariable_inverse_transform() -> None: + rv = uq.UniformRandomVariable('test_rv', theta=np.array((0.0, 1.0))) samples = np.array((0.10, 0.20, 0.30)) rv.uni_sample = samples rv.inverse_transform_sampling() - inverse_transform = rv.sample + inverse_transform = ensure_value(rv.sample) assert np.allclose(inverse_transform, samples, rtol=1e-5) # # uniform with unspecified bounds # - rv = uq.UniformRandomVariable('test_rv', theta=(np.nan, 1.0)) + rv = uq.UniformRandomVariable('test_rv', theta=np.array((np.nan, 1.0))) samples = np.array((0.10, 0.20, 0.30)) rv.uni_sample = samples rv.inverse_transform_sampling() - inverse_transform = rv.sample + inverse_transform = ensure_value(rv.sample) assert np.all(np.isnan(inverse_transform)) - rv = uq.UniformRandomVariable('test_rv', theta=(0.00, np.nan)) + rv = uq.UniformRandomVariable('test_rv', theta=np.array((0.00, np.nan))) rv.uni_sample = samples rv.inverse_transform_sampling() - inverse_transform = rv.sample + inverse_transform = ensure_value(rv.sample) assert np.all(np.isinf(inverse_transform)) rv = uq.UniformRandomVariable( 'test_rv', - theta=(0.00, 1.00), + theta=np.array((0.00, 1.00)), truncation_limits=np.array((0.20, 0.80)), ) rv.uni_sample = samples rv.inverse_transform_sampling() - inverse_transform = rv.sample + inverse_transform = ensure_value(rv.sample) assert np.allclose(inverse_transform, np.array((0.26, 0.32, 0.38)), rtol=1e-5) # sample as a pandas series, with a log() map rv.f_map = np.log - assert rv.sample_DF.to_dict() == { + assert ensure_value(rv.sample_DF).to_dict() == { 0: -1.3470736479666092, 1: -1.1394342831883646, 2: -0.9675840262617056, @@ -1168,57 +1346,58 @@ def test_UniformRandomVariable_inverse_transform(): # # uniform without values to sample from - rv = uq.UniformRandomVariable('test_rv', theta=(0.0, 1.0)) - with pytest.raises(ValueError): + rv = uq.UniformRandomVariable('test_rv', theta=np.array((0.0, 1.0))) + with pytest.raises(ValueError, match='No available uniform sample.'): rv.inverse_transform_sampling() -def test_WeibullRandomVariable(): +def test_WeibullRandomVariable() -> None: rv = uq.WeibullRandomVariable('rv_name', theta=np.array((1.5, 2.0))) assert rv.name == 'rv_name' + assert rv.theta is not None np.testing.assert_allclose(rv.theta, np.array((1.5, 2.0))) assert np.all(np.isnan(rv.truncation_limits)) assert rv.RV_set is None assert rv.sample_DF is None with pytest.raises(AttributeError): - rv.xyz = 123 + rv.xyz = 123 # type: ignore -def test_WeibullRandomVariable_cdf(): +def test_WeibullRandomVariable_cdf() -> None: rv = uq.WeibullRandomVariable( 'test_rv', - theta=(1.5, 2.0), + theta=np.array((1.5, 2.0)), truncation_limits=np.array((0.5, 2.5)), ) - x = (0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0) + x = np.array((0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0)) cdf = rv.cdf(x) expected_cdf = np.array([0.0, 0.0, 0.30463584, 0.63286108, 0.87169261, 1.0, 1.0]) assert np.allclose(cdf, expected_cdf, rtol=1e-5) - rv = uq.WeibullRandomVariable('test_rv', theta=(1.5, 2.0)) + rv = uq.WeibullRandomVariable('test_rv', theta=np.array((1.5, 2.0))) cdf = rv.cdf(x) expected_cdf_no_trunc = weibull_min.cdf(x, 2.0, scale=1.5) assert np.allclose(cdf, expected_cdf_no_trunc, rtol=1e-5) -def test_WeibullRandomVariable_inverse_transform(): +def test_WeibullRandomVariable_inverse_transform() -> None: samples = np.array((0.10, 0.20, 0.30)) - rv = uq.WeibullRandomVariable('test_rv', theta=(1.5, 2.0)) + rv = uq.WeibullRandomVariable('test_rv', theta=np.array((1.5, 2.0))) rv.uni_sample = samples rv.inverse_transform_sampling() - inverse_transform = rv.sample + inverse_transform = ensure_value(rv.sample) expected_samples = weibull_min.ppf(samples, 2.0, scale=1.5) assert np.allclose(inverse_transform, expected_samples, rtol=1e-5) rv = uq.WeibullRandomVariable( - 'test_rv', theta=(1.5, 2.0), truncation_limits=(0.5, 2.5) + 'test_rv', theta=np.array((1.5, 2.0)), truncation_limits=np.array((0.5, 2.5)) ) rv.uni_sample = samples rv.inverse_transform_sampling() - inverse_transform = rv.sample + inverse_transform = ensure_value(rv.sample) truncated_samples = weibull_min.ppf( samples * ( @@ -1232,64 +1411,97 @@ def test_WeibullRandomVariable_inverse_transform(): assert np.allclose(inverse_transform, truncated_samples, rtol=1e-5) -def test_MultinomialRandomVariable(): +def test_MultinomialRandomVariable() -> None: # multinomial with invalid p values provided in the theta vector - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match=re.escape( + 'The set of p values provided for a multinomial ' + 'distribution shall sum up to less than or equal to 1.0. ' + 'The provided values sum up to 43.0. ' + 'p = [ 0.2 0.7 0.1 42. ] .' + ), + ): uq.MultinomialRandomVariable( 'rv_invalid', np.array((0.20, 0.70, 0.10, 42.00)) ) -def test_MultilinearCDFRandomVariable(): +def test_MultilinearCDFRandomVariable() -> None: # multilinear CDF: cases that should fail x_values = (0.00, 1.00, 2.00, 3.00, 4.00) y_values = (100.00, 0.20, 0.20, 0.80, 1.00) values = np.column_stack((x_values, y_values)) - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match='For multilinear CDF random variables, y_1 should be set to 0.00', + ): uq.MultilinearCDFRandomVariable('test_rv', theta=values) x_values = (0.00, 1.00, 2.00, 3.00, 4.00) y_values = (0.00, 0.20, 0.20, 0.80, 0.80) values = np.column_stack((x_values, y_values)) - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match='For multilinear CDF random variables, y_n should be set to 1.00', + ): uq.MultilinearCDFRandomVariable('test_rv', theta=values) x_values = (0.00, 3.00, 1.00, 2.00, 4.00) y_values = (0.00, 0.25, 0.50, 0.75, 1.00) values = np.column_stack((x_values, y_values)) - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match='For multilinear CDF random variables, Xs should be specified in ascending order', + ): uq.MultilinearCDFRandomVariable('test_rv', theta=values) x_values = (0.00, 1.00, 2.00, 3.00, 4.00) y_values = (0.00, 0.75, 0.50, 0.25, 1.00) values = np.column_stack((x_values, y_values)) - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match='For multilinear CDF random variables, Ys should be specified in ascending order', + ): uq.MultilinearCDFRandomVariable('test_rv', theta=values) x_values = (0.00, 1.00, 2.00, 3.00, 4.00) y_values = (0.00, 0.50, 0.50, 0.50, 1.00) values = np.column_stack((x_values, y_values)) - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match=( + 'For multilinear CDF random variables, ' + 'Ys should be specified in strictly ascending order' + ), + ): uq.MultilinearCDFRandomVariable('test_rv', theta=values) x_values = (0.00, 2.00, 2.00, 3.00, 4.00) y_values = (0.00, 0.20, 0.40, 0.50, 1.00) values = np.column_stack((x_values, y_values)) - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match=( + 'For multilinear CDF random variables, ' + 'Xs should be specified in strictly ascending order' + ), + ): uq.MultilinearCDFRandomVariable('test_rv', theta=values) -def test_MultilinearCDFRandomVariable_cdf(): +def test_MultilinearCDFRandomVariable_cdf() -> None: x_values = (0.00, 1.00, 2.00, 3.00, 4.00) y_values = (0.00, 0.20, 0.30, 0.80, 1.00) values = np.column_stack((x_values, y_values)) rv = uq.MultilinearCDFRandomVariable('test_rv', theta=values) # confirm that creating an attribute on the fly is not allowed with pytest.raises(AttributeError): - # pylint: disable=assigning-non-slot - rv.xyz = 123 - x = (-100.00, 0.00, 0.50, 1.00, 1.50, 2.00, 2.50, 3.00, 3.50, 4.00, 100.00) + rv.xyz = 123 # type: ignore + x = np.array( + (-100.00, 0.00, 0.50, 1.00, 1.50, 2.00, 2.50, 3.00, 3.50, 4.00, 100.00) + ) cdf = rv.cdf(x) assert np.allclose( @@ -1299,7 +1511,7 @@ def test_MultilinearCDFRandomVariable_cdf(): ) -def test_MultilinearCDFRandomVariable_inverse_transform(): +def test_MultilinearCDFRandomVariable_inverse_transform() -> None: x_values = (0.00, 1.00, 2.00, 3.00, 4.00) y_values = (0.00, 0.20, 0.30, 0.80, 1.00) values = np.column_stack((x_values, y_values)) @@ -1307,7 +1519,7 @@ def test_MultilinearCDFRandomVariable_inverse_transform(): rv.uni_sample = np.array((0.00, 0.1, 0.2, 0.5, 0.8, 0.9, 1.00)) rv.inverse_transform_sampling() - inverse_transform = rv.sample + inverse_transform = ensure_value(rv.sample) assert np.allclose( inverse_transform, np.array((0.00, 0.50, 1.00, 2.40, 3.00, 3.50, 4.00)), @@ -1315,100 +1527,95 @@ def test_MultilinearCDFRandomVariable_inverse_transform(): ) -def test_EmpiricalRandomVariable_inverse_transform(): +def test_EmpiricalRandomVariable_inverse_transform() -> None: samples = np.array((0.10, 0.20, 0.30)) rv_empirical = uq.EmpiricalRandomVariable( - 'test_rv_empirical', raw_samples=(1.00, 2.00, 3.00, 4.00) + 'test_rv_empirical', theta=np.array((1.00, 2.00, 3.00, 4.00)) ) # confirm that creating an attribute on the fly is not allowed with pytest.raises(AttributeError): - # pylint: disable=assigning-non-slot - rv_empirical.xyz = 123 + rv_empirical.xyz = 123 # type: ignore samples = np.array((0.10, 0.50, 0.90)) rv_empirical.uni_sample = samples rv_empirical.inverse_transform_sampling() - inverse_transform = rv_empirical.sample + inverse_transform = ensure_value(rv_empirical.sample) assert np.allclose(inverse_transform, np.array((1.00, 3.00, 4.00)), rtol=1e-5) rv_coupled = uq.CoupledEmpiricalRandomVariable( 'test_rv_coupled', - raw_samples=np.array((1.00, 2.00, 3.00, 4.00)), + theta=np.array((1.00, 2.00, 3.00, 4.00)), ) rv_coupled.inverse_transform_sampling(sample_size=6) - inverse_transform = rv_coupled.sample + inverse_transform = ensure_value(rv_coupled.sample) assert np.allclose( inverse_transform, np.array((1.00, 2.00, 3.00, 4.00, 1.00, 2.00)), rtol=1e-5 ) -def test_DeterministicRandomVariable_inverse_transform(): +def test_DeterministicRandomVariable_inverse_transform() -> None: rv = uq.DeterministicRandomVariable('test_rv', theta=np.array((0.00,))) rv.inverse_transform_sampling(4) - inverse_transform = rv.sample + inverse_transform = ensure_value(rv.sample) assert np.allclose( inverse_transform, np.array((0.00, 0.00, 0.00, 0.00)), rtol=1e-5 ) -def test_RandomVariable_Set(): +def test_RandomVariable_Set() -> None: # a set of two random variables - rv_1 = uq.NormalRandomVariable('rv1', theta=(1.0, 1.0)) - rv_2 = uq.NormalRandomVariable('rv2', theta=(1.0, 1.0)) - rv_set = uq.RandomVariableSet( # noqa: F841 - 'test_set', (rv_1, rv_2), np.array(((1.0, 0.50), (0.50, 1.0))) + rv_1 = uq.NormalRandomVariable('rv1', theta=np.array((1.0, 1.0))) + rv_2 = uq.NormalRandomVariable('rv2', theta=np.array((1.0, 1.0))) + rv_set = uq.RandomVariableSet( + 'test_set', [rv_1, rv_2], np.array(((1.0, 0.50), (0.50, 1.0))) ) # size of the set assert rv_set.size == 2 # a set with only one random variable - rv_1 = uq.NormalRandomVariable('rv1', theta=(1.0, 1.0)) - rv_set = uq.RandomVariableSet( # noqa: F841 - 'test_set', (rv_1,), np.array(((1.0, 0.50),)) - ) + rv_1 = uq.NormalRandomVariable('rv1', theta=np.array((1.0, 1.0))) + rv_set = uq.RandomVariableSet('test_set', [rv_1], np.array(((1.0, 0.50),))) -def test_RandomVariable_perfect_correlation(): - +def test_RandomVariable_perfect_correlation() -> None: # Test that the `.anchor` attribute is propagated correctly - rv_1 = uq.NormalRandomVariable('rv1', theta=(1.0, 1.0)) - rv_2 = uq.NormalRandomVariable('rv2', theta=(1.0, 1.0), anchor=rv_1) + rv_1 = uq.NormalRandomVariable('rv1', theta=np.array((1.0, 1.0))) + rv_2 = uq.NormalRandomVariable('rv2', theta=np.array((1.0, 1.0)), anchor=rv_1) rv_1.uni_sample = np.random.random(size=10) assert np.all(rv_2.uni_sample == rv_1.uni_sample) - rv_1 = uq.NormalRandomVariable('rv1', theta=(1.0, 1.0)) - rv_2 = uq.NormalRandomVariable('rv2', theta=(1.0, 1.0)) + rv_1 = uq.NormalRandomVariable('rv1', theta=np.array((1.0, 1.0))) + rv_2 = uq.NormalRandomVariable('rv2', theta=np.array((1.0, 1.0))) rv_1.uni_sample = np.random.random(size=10) assert rv_2.uni_sample is None -def test_RandomVariable_Set_apply_correlation(reset=False): +def test_RandomVariable_Set_apply_correlation(*, reset: bool = False) -> None: data_dir = ( 'pelicun/tests/basic/data/uq/test_random_variable_set_apply_correlation' ) - file_incr = 0 # correlated, uniform np.random.seed(40) - rv_1 = uq.UniformRandomVariable(name='rv1', theta=(-5.0, 5.0)) - rv_2 = uq.UniformRandomVariable(name='rv2', theta=(-5.0, 5.0)) + rv_1 = uq.UniformRandomVariable(name='rv1', theta=np.array((-5.0, 5.0))) + rv_2 = uq.UniformRandomVariable(name='rv2', theta=np.array((-5.0, 5.0))) rv_1.uni_sample = np.random.random(size=100) rv_2.uni_sample = np.random.random(size=100) rvs = uq.RandomVariableSet( - name='test_set', RV_list=[rv_1, rv_2], Rho=np.array(((1.0, 0.5), (0.5, 1.0))) + name='test_set', rv_list=[rv_1, rv_2], rho=np.array(((1.0, 0.5), (0.5, 1.0))) ) rvs.apply_correlation() - for rv in (rv_1, rv_2): - res = rv.uni_sample - file_incr += 1 + for i, rv in enumerate((rv_1, rv_2)): + res = ensure_value(rv.uni_sample) + file_incr = i + 1 filename = f'{data_dir}/test_{file_incr}.pcl' if reset: export_pickle(filename, res) @@ -1420,13 +1627,13 @@ def test_RandomVariable_Set_apply_correlation(reset=False): rv_1.inverse_transform_sampling() rv_2.inverse_transform_sampling() rvset_sample = rvs.sample - assert set(rvset_sample.keys()) == set(('rv1', 'rv2')) + assert set(rvset_sample.keys()) == {'rv1', 'rv2'} vals = list(rvset_sample.values()) assert np.all(vals[0] == rv_1.sample) assert np.all(vals[1] == rv_2.sample) -def test_RandomVariable_Set_apply_correlation_special(): +def test_RandomVariable_Set_apply_correlation_special() -> None: # This function tests the apply_correlation method of the # RandomVariableSet class when given special input conditions. # The first test checks that the method works when given a non @@ -1442,8 +1649,8 @@ def test_RandomVariable_Set_apply_correlation_special(): # non positive semidefinite correlation matrix rho = np.array(((1.00, 0.50), (0.50, -1.00))) - rv_1 = uq.NormalRandomVariable('rv1', theta=[5.0, 0.1]) - rv_2 = uq.NormalRandomVariable('rv2', theta=[5.0, 0.1]) + rv_1 = uq.NormalRandomVariable('rv1', theta=np.array((5.0, 0.1))) + rv_2 = uq.NormalRandomVariable('rv2', theta=np.array((5.0, 0.1))) rv_1.uni_sample = np.random.random(size=100) rv_2.uni_sample = np.random.random(size=100) rv_set = uq.RandomVariableSet('rv_set', [rv_1, rv_2], rho) @@ -1451,8 +1658,8 @@ def test_RandomVariable_Set_apply_correlation_special(): # non full rank matrix rho = np.array(((0.00, 0.00), (0.0, 0.0))) - rv_1 = uq.NormalRandomVariable('rv1', theta=[5.0, 0.1]) - rv_2 = uq.NormalRandomVariable('rv2', theta=[5.0, 0.1]) + rv_1 = uq.NormalRandomVariable('rv1', theta=np.array((5.0, 0.1))) + rv_2 = uq.NormalRandomVariable('rv2', theta=np.array((5.0, 0.1))) rv_1.uni_sample = np.random.random(size=100) rv_2.uni_sample = np.random.random(size=100) rv_set = uq.RandomVariableSet('rv_set', [rv_1, rv_2], rho) @@ -1462,23 +1669,23 @@ def test_RandomVariable_Set_apply_correlation_special(): ) -def test_RandomVariable_Set_orthotope_density(reset=False): +def test_RandomVariable_Set_orthotope_density(*, reset: bool = False) -> None: data_dir = ( 'pelicun/tests/basic/data/uq/test_random_variable_set_orthotope_density' ) # create some random variables rv_1 = uq.Normal_COV( - 'rv1', theta=[5.0, 0.1], truncation_limits=np.array((np.nan, 10.0)) + 'rv1', theta=np.array((5.0, 0.1)), truncation_limits=np.array((np.nan, 10.0)) ) - rv_2 = uq.LogNormalRandomVariable('rv2', theta=[10.0, 0.2]) - rv_3 = uq.UniformRandomVariable('rv3', theta=[13.0, 17.0]) - rv_4 = uq.UniformRandomVariable('rv4', theta=[0.0, 1.0]) - rv_5 = uq.UniformRandomVariable('rv5', theta=[0.0, 1.0]) + rv_2 = uq.LogNormalRandomVariable('rv2', theta=np.array((10.0, 0.2))) + rv_3 = uq.UniformRandomVariable('rv3', theta=np.array((13.0, 17.0))) + rv_4 = uq.UniformRandomVariable('rv4', theta=np.array((0.0, 1.0))) + rv_5 = uq.UniformRandomVariable('rv5', theta=np.array((0.0, 1.0))) # create a random variable set rv_set = uq.RandomVariableSet( - 'rv_set', (rv_1, rv_2, rv_3, rv_4, rv_5), np.identity(5) + 'rv_set', [rv_1, rv_2, rv_3, rv_4, rv_5], np.identity(5) ) # define test cases @@ -1487,7 +1694,7 @@ def test_RandomVariable_Set_orthotope_density(reset=False): ( np.array([4.0, 9.0, 14.0, np.nan]), np.array([6.0, 11.0, 16.0, 0.80]), - ('rv1', 'rv2', 'rv3', 'rv4'), + ['rv1', 'rv2', 'rv3', 'rv4'], ), ( np.array([4.0, 9.0, 14.0, np.nan, 0.20]), @@ -1522,7 +1729,7 @@ def test_RandomVariable_Set_orthotope_density(reset=False): assert np.allclose(res, compare) -def test_RandomVariableRegistry_generate_sample(reset=False): +def test_RandomVariableRegistry_generate_sample(*, reset: bool = False) -> None: data_dir = ( 'pelicun/tests/basic/data/uq/test_RandomVariableRegistry_generate_sample' ) @@ -1537,14 +1744,14 @@ def test_RandomVariableRegistry_generate_sample(reset=False): rng = np.random.default_rng(0) rv_registry_single = uq.RandomVariableRegistry(rng) # create the random variable and add it to the registry - RV = uq.NormalRandomVariable('x', theta=[1.0, 1.0]) - rv_registry_single.add_RV(RV) + rv = uq.NormalRandomVariable('x', theta=np.array((1.0, 1.0))) + rv_registry_single.add_RV(rv) # Generate a sample sample_size = 1000 rv_registry_single.generate_sample(sample_size, method) - res = rv_registry_single.RV_sample['x'] + res = ensure_value(rv_registry_single.RV_sample['x']) assert len(res) == sample_size file_incr += 1 @@ -1566,13 +1773,15 @@ def test_RandomVariableRegistry_generate_sample(reset=False): # create a random variable registry and add some random variables to it rng = np.random.default_rng(4) rv_registry = uq.RandomVariableRegistry(rng) - rv_1 = uq.Normal_COV('rv1', theta=[5.0, 0.1]) - rv_2 = uq.LogNormalRandomVariable('rv2', theta=[10.0, 0.2]) - rv_3 = uq.UniformRandomVariable('rv3', theta=[13.0, 17.0]) + rv_1 = uq.Normal_COV('rv1', theta=np.array((5.0, 0.1))) + rv_2 = uq.LogNormalRandomVariable('rv2', theta=np.array((10.0, 0.2))) + rv_3 = uq.UniformRandomVariable('rv3', theta=np.array((13.0, 17.0))) rv_registry.add_RV(rv_1) rv_registry.add_RV(rv_2) rv_registry.add_RV(rv_3) - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match='RV rv3 already exists in the registry.' + ): rv_registry.add_RV(rv_3) # create a random variable set and add it to the registry @@ -1582,8 +1791,8 @@ def test_RandomVariableRegistry_generate_sample(reset=False): rv_registry.add_RV_set(rv_set) # add some more random variables that are not part of the set - rv_4 = uq.Normal_COV('rv4', theta=[14.0, 0.30]) - rv_5 = uq.Normal_COV('rv5', theta=[15.0, 0.50]) + rv_4 = uq.Normal_COV('rv4', theta=np.array((14.0, 0.30))) + rv_5 = uq.Normal_COV('rv5', theta=np.array((15.0, 0.50))) rv_registry.add_RV(rv_4) rv_registry.add_RV(rv_5) @@ -1591,7 +1800,7 @@ def test_RandomVariableRegistry_generate_sample(reset=False): # verify that all samples have been generated as expected for rv_name in (f'rv{i + 1}' for i in range(5)): - res = rv_registry.RV_sample[rv_name] + res = ensure_value(rv_registry.RV_sample[rv_name]) file_incr += 1 filename = f'{data_dir}/test_{file_incr}.pcl' if reset: @@ -1600,17 +1809,19 @@ def test_RandomVariableRegistry_generate_sample(reset=False): assert np.allclose(res, compare) # obtain multiple RVs from the registry - rv_dictionary = rv_registry.RVs(('rv1', 'rv2')) + rv_dictionary = rv_registry.RVs(['rv1', 'rv2']) assert 'rv1' in rv_dictionary assert 'rv2' in rv_dictionary assert 'rv3' not in rv_dictionary -def test_rv_class_map(): +def test_rv_class_map() -> None: rv_class = uq.rv_class_map('normal_std') assert rv_class.__name__ == 'Normal_STD' - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match=re.escape('Unsupported distribution: ') + ): uq.rv_class_map('') diff --git a/pelicun/tests/code_repetition_checker.py b/pelicun/tests/code_repetition_checker.py index 3bf66f4f6..d877e4b2a 100644 --- a/pelicun/tests/code_repetition_checker.py +++ b/pelicun/tests/code_repetition_checker.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -44,23 +43,27 @@ """ from __future__ import annotations -from glob2 import glob # type: ignore -# pylint: disable=missing-any-param-doc +from pathlib import Path + +from glob2 import glob # type: ignore -def main(file): +def main(file: str) -> None: """ Identifies and displays repeated consecutive line blocks within a file, including their line numbers. - Args: - file: Path to the file to be checked for duplicates. + Parameters + ---------- + file: str + Path to the file to be checked for duplicates. + """ # file = 'tests/test_uq.py' group = 15 # find repeated blocks this many lines - with open(file, 'r', encoding='utf-8') as f: + with Path(file).open(encoding='utf-8') as f: contents = f.readlines() num_lines = len(contents) for i in range(0, num_lines, group): @@ -68,22 +71,22 @@ def main(file): for j in range(i + 1, num_lines): jlines = contents[j : j + group] if glines == jlines: - print(f'{i, j}: ') + print(f'{i, j}: ') # noqa: T201 for k in range(group): - print(f' {jlines[k]}', end='') - print() + print(f' {jlines[k]}', end='') # noqa: T201 + print() # noqa: T201 -def all_test_files(): +def all_test_files() -> None: """ Searches for all Python test files in the 'tests' directory and runs the main function to find and print repeated line blocks in each file. """ test_files = glob('tests/*.py') for file in test_files: - print() - print(file) - print() + print() # noqa: T201 + print(file) # noqa: T201 + print() # noqa: T201 main(file) diff --git a/pelicun/tests/compatibility/PRJ-3411v5/CMP_marginals.csv b/pelicun/tests/compatibility/PRJ-3411v5/CMP_marginals.csv deleted file mode 100755 index 13c141e77..000000000 --- a/pelicun/tests/compatibility/PRJ-3411v5/CMP_marginals.csv +++ /dev/null @@ -1,38 +0,0 @@ -,Units,Location,Direction,Theta_0,Blocks,Comment -B.10.41.001a,ea,"3, 4","1,2",2,,"24x24 ACI 318 SMF, beam on one side" -B.10.41.002a,ea,1,"1,2",2,,"24x36 ACI 318 SMF, beam on one side" -B.10.41.002a,ea,2,"1,2",1,,"24x36 ACI 318 SMF, beam on one side" -B.10.41.002b,ea,2--4,"1,2",3,,"24x36 ACI 318 SMF, beam on both sides" -B.10.41.003a,ea,2,"1,2",1,,"36x36 ACI 318 SMF, beam on one side" -B.10.41.003b,ea,1,"1,2",3,,"36x36 ACI 318 SMF, beam on both sides" -B.10.49.031,ea,all,0,15,,Post-tensioned concrete flat slabs- columns with shear reinforcing 0 2.5 inches), SDC D,E,F, PIPING FRAGILITY" -D.20.21.023b,ft,all,0,1230,,"Cold or Hot Potable Water Piping (dia > 2.5 inches), SDC D,E,F, BRACING FRAGILITY" -D.20.31.013b,ft,all,0,1230,,"Sanitary Waste Piping - Cast Iron w/flexible couplings, SDC D,E,F, BRACING FRAGILITY" -D.30.41.021c,ft,all,0,1620,,"HVAC Stainless Steel Ducting less than 6 sq. ft in cross sectional area, SDC D, E, or F" -D.30.41.022c,ft,all,0,430,,"HVAC Stainless Steel Ducting - 6 sq. ft cross sectional area or greater, SDC D, E, or F" -D.30.41.032c,ea,all,0,19.44,,"HVAC Drops / Diffusers without ceilings - supported by ducting only - No independent safety wires, SDC D, E, or F" -D.30.41.041b,ea,all,0,16,,"Variable Air Volume (VAV) box with in-line coil, SDC C" -D.20.61.013b,ft,all,0,1920,,"Steam Piping - Small Diameter Threaded Steel - (2.5 inches in diameter or less), SDC D, E, or F, BRACING FRAGILITY" -D.20.22.013a,ft,all,0,1920,,"Heating hot Water Piping - Small Diameter Threaded Steel - (2.5 inches in diameter or less), SDC D, E, or F, PIPING FRAGILITY" -D.20.22.023a,ft,all,0,760,,"Heating hot Water Piping - Large Diameter Welded Steel - (greater than 2.5 inches in diameter), SDC D, E, or F, PIPING FRAGILITY" -D.20.22.023b,ft,all,0,760,,"Heating hot Water Piping - Large Diameter Welded Steel - (greater than 2.5 inches in diameter), SDC D, E, or F, BRACING FRAGILITY" -D.40.11.033a,ea,all,0,194,,"Fire Sprinkler Drop Standard Threaded Steel - Dropping into unbraced lay-in tile SOFT ceiling - 6 ft. long drop maximum, SDC D, E, or F" -B.30.11.011,ft2,roof,0,5832,,"Concrete tile roof, tiles secured and compliant with UBC94" -D.30.31.013i,ea,roof,0,1,,Chiller - Capacity: 350 to <750 Ton - Equipment that is either hard anchored or is vibration isolated with seismic snubbers/restraints - Combined anchorage/isolator & equipment fragility -D.30.31.023i,ea,roof,0,1,,Cooling Tower - Capacity: 350 to <750 Ton - Equipment that is either hard anchored or is vibration isolated with seismic snubbers/restraints - Combined anchorage/isolator & equipment fragility -D.30.52.013i,ea,roof,0,4,,Air Handling Unit - Capacity: 10000 to <25000 CFM - Equipment that is either hard anchored or is vibration isolated with seismic snubbers/restraints - Combined anchorage/isolator & equipment fragility -excessiveRID,ea,all,"1,2",1,,Excessive residual drift -collapse,ea,0,1,1,,Collapsed building -irreparable,ea,0,1,1,,Irreparable building diff --git a/pelicun/tests/compatibility/PRJ-3411v5/demand_data.csv b/pelicun/tests/compatibility/PRJ-3411v5/demand_data.csv deleted file mode 100755 index df02e850e..000000000 --- a/pelicun/tests/compatibility/PRJ-3411v5/demand_data.csv +++ /dev/null @@ -1,145 +0,0 @@ -,median,log_std -1-PFA-0-1,0.08,0.4608067551638503 -1-PFA-0-2,0.08,0.49413271857642355 -1-PFA-1-1,0.14,0.4681344734414342 -1-PFA-1-2,0.15,0.2868530450370383 -1-PFA-2-1,0.17,0.4164624252641039 -1-PFA-2-2,0.17,0.4164624252641039 -1-PFA-3-1,0.18,0.40511321955375623 -1-PFA-3-2,0.18,0.40511321955375623 -1-PFA-4-1,0.23,0.34726426562687884 -1-PFA-4-2,0.23,0.3547900357092975 -1-PID-1-1,0.005,0.4270757253801016 -1-PID-1-2,0.005,0.4270757253801016 -1-PID-2-1,0.005,0.4270757253801016 -1-PID-2-2,0.005,0.4270757253801016 -1-PID-3-1,0.004,0.4333520014182283 -1-PID-3-2,0.004,0.4333520014182283 -1-PID-4-1,0.002,0.5 -1-PID-4-2,0.002,0.5 -2-PFA-0-1,0.21,0.40963505144687823 -2-PFA-0-2,0.21,0.3832845165220612 -2-PFA-1-1,0.29,0.4288796990711922 -2-PFA-1-2,0.29,0.4288796990711922 -2-PFA-2-1,0.28,0.3876227323608682 -2-PFA-2-2,0.29,0.40099778751361925 -2-PFA-3-1,0.3,0.4035091985429514 -2-PFA-3-2,0.27,0.3997088915138062 -2-PFA-4-1,0.33,0.37239657187771463 -2-PFA-4-2,0.32,0.4209585845343814 -2-PID-1-1,0.011,0.4711097456140563 -2-PID-1-2,0.01,0.475131730518475 -2-PID-2-1,0.012,0.4394939456771386 -2-PID-2-2,0.011,0.4711097456140563 -2-PID-3-1,0.008,0.5 -2-PID-3-2,0.008,0.447946327171986 -2-PID-4-1,0.004,0.4333520014182283 -2-PID-4-2,0.003,0.5 -3-PFA-0-1,0.34,0.4205559989300818 -3-PFA-0-2,0.33,0.37035195963979445 -3-PFA-1-1,0.41,0.3772360727062722 -3-PFA-1-2,0.42,0.3757655292605639 -3-PFA-2-1,0.36,0.37647176413422945 -3-PFA-2-2,0.38,0.40508979107103643 -3-PFA-3-1,0.35,0.37539754064726244 -3-PFA-3-2,0.35,0.38276503265877987 -3-PFA-4-1,0.43,0.3817574009955636 -3-PFA-4-2,0.41,0.39375144764296094 -3-PID-1-1,0.016,0.44795243307594346 -3-PID-1-2,0.017,0.5222864082152147 -3-PID-2-1,0.018,0.4027931427882727 -3-PID-2-2,0.019,0.458870056460066 -3-PID-3-1,0.014,0.4231794130494331 -3-PID-3-2,0.014,0.45842683288503877 -3-PID-4-1,0.006,0.4203185874397092 -3-PID-4-2,0.006,0.6135685945427505 -4-PFA-0-1,0.49,0.3919210430750446 -4-PFA-0-2,0.45,0.38698384475251535 -4-PFA-1-1,0.53,0.406011263972564 -4-PFA-1-2,0.5,0.38752352095832543 -4-PFA-2-1,0.45,0.39274739627448507 -4-PFA-2-2,0.46,0.3945545500174033 -4-PFA-3-1,0.4,0.38127939790214715 -4-PFA-3-2,0.41,0.3772360727062722 -4-PFA-4-1,0.49,0.33956788384728936 -4-PFA-4-2,0.49,0.37524568788735635 -4-PID-1-1,0.026,0.5124532035318107 -4-PID-1-2,0.027,0.5970969478920988 -4-PID-2-1,0.027,0.4794717270402726 -4-PID-2-2,0.027,0.5689226436084314 -4-PID-3-1,0.02,0.3959232466570905 -4-PID-3-2,0.02,0.50639149262782 -4-PID-4-1,0.008,0.5184667123718524 -4-PID-4-2,0.008,0.6444856071246744 -5-PFA-0-1,0.53,0.49558086996772627 -5-PFA-0-2,0.52,0.3888274185466733 -5-PFA-1-1,0.6,0.4327984859794568 -5-PFA-1-2,0.58,0.40300677399524637 -5-PFA-2-1,0.45,0.39859880249384605 -5-PFA-2-2,0.46,0.4302023973054347 -5-PFA-3-1,0.42,0.3894423471436558 -5-PFA-3-2,0.43,0.4045390070382661 -5-PFA-4-1,0.5,0.37083251851057364 -5-PFA-4-2,0.51,0.3922488768771692 -5-PID-1-1,0.034,0.4937862195749691 -5-PID-1-2,0.029,0.45655108499263214 -5-PID-2-1,0.035,0.4719298354100396 -5-PID-2-2,0.03,0.4712487198187546 -5-PID-3-1,0.021,0.5390460626558675 -5-PID-3-2,0.025,0.4264982747013112 -5-PID-4-1,0.008,0.742315433526588 -5-PID-4-2,0.012,0.5 -6-PFA-0-1,0.67,0.433673896418087 -6-PFA-0-2,0.65,0.4028467214673338 -6-PFA-1-1,0.72,0.4124720061431207 -6-PFA-1-2,0.66,0.4145767084859249 -6-PFA-2-1,0.52,0.3618066495006511 -6-PFA-2-2,0.54,0.4126302775358337 -6-PFA-3-1,0.46,0.40667200135708653 -6-PFA-3-2,0.47,0.3765029041603888 -6-PFA-4-1,0.53,0.3687296645707586 -6-PFA-4-2,0.55,0.42854198175786573 -6-PID-1-1,0.038,0.499774428013506 -6-PID-1-2,0.037,0.5712444846665292 -6-PID-2-1,0.039,0.4594344469648642 -6-PID-2-2,0.04,0.49413089340948024 -6-PID-3-1,0.02,0.5414775471204692 -6-PID-3-2,0.027,0.4835005040366075 -6-PID-4-1,0.009,0.7538901102273311 -6-PID-4-2,0.014,0.7589902208022125 -7-PFA-0-1,0.87,0.44120648819447855 -7-PFA-0-2,0.78,0.4177407092067101 -7-PFA-1-1,0.86,0.4821081396301719 -7-PFA-1-2,0.82,0.4006453426541226 -7-PFA-2-1,0.56,0.34705572162101234 -7-PFA-2-2,0.62,0.40820261049405293 -7-PFA-3-1,0.57,0.4232287075991503 -7-PFA-3-2,0.6,0.39420276076969885 -7-PFA-4-1,0.6,0.36658338302347926 -7-PFA-4-2,0.7,0.458458353356435 -7-PID-1-1,0.048,0.5890989181011332 -7-PID-1-2,0.035,0.4489672960554259 -7-PID-2-1,0.05,0.5091893954559619 -7-PID-2-2,0.038,0.46582524029968414 -7-PID-3-1,0.024,0.5880264933787529 -7-PID-3-2,0.033,0.5640052852568394 -7-PID-4-1,0.013,0.8379017172040789 -7-PID-4-2,0.025,0.802097323851652 -8-PFA-0-1,0.95,0.3567116524771788 -8-PFA-0-2,1.1,0.3470735728935639 -8-PFA-1-1,1.05,0.42608849302357177 -8-PFA-1-2,0.76,0.5038492195451799 -8-PFA-2-1,0.6,0.36811806938860964 -8-PFA-2-2,0.66,0.3596707400773089 -8-PFA-3-1,0.6,0.5810192984266385 -8-PFA-3-2,0.46,0.375216650813277 -8-PFA-4-1,0.62,0.42130837935001453 -8-PFA-4-2,0.62,0.37154410493248724 -8-PID-1-1,0.064,0.37225108957396674 -8-PID-1-2,0.034,0.5541025203432555 -8-PID-2-1,0.065,0.38772639486747945 -8-PID-2-2,0.036,0.5609854356653375 -8-PID-3-1,0.037,0.4291250037192124 -8-PID-3-2,0.028,0.42325787086810823 -8-PID-4-1,0.018,0.3654463171640024 -8-PID-4-2,0.016,0.7337218554010868 diff --git a/pelicun/tests/compatibility/PRJ-3411v5/test_compatibility_prj3411v5.py b/pelicun/tests/compatibility/PRJ-3411v5/test_compatibility_prj3411v5.py deleted file mode 100644 index 7f7076454..000000000 --- a/pelicun/tests/compatibility/PRJ-3411v5/test_compatibility_prj3411v5.py +++ /dev/null @@ -1,207 +0,0 @@ -""" -This test verifies backwards compatibility with the following: - - DesignSafe PRJ-3411 > Example01_FEMA_P58_Introduction - -There are some changes made to the code and input files, so the output -is not the same with what the original code would produce. We only -want to confirm that executing this code does not raise an error. - -""" - -from __future__ import annotations -import numpy as np -import pandas as pd -import pytest -from pelicun.warnings import PelicunWarning -from pelicun.base import convert_to_MultiIndex -from pelicun.assessment import Assessment - - -def test_compatibility_DesignSafe_PRJ_3411_Example01(): - - sample_size = 10000 - raw_demands = pd.read_csv( - 'pelicun/tests/compatibility/PRJ-3411v5/demand_data.csv', index_col=0 - ) - raw_demands = convert_to_MultiIndex(raw_demands, axis=0) - raw_demands.index.names = ['stripe', 'type', 'loc', 'dir'] - stripe = '3' - stripe_demands = raw_demands.loc[stripe, :] - stripe_demands.insert(0, 'Units', "") - stripe_demands.loc['PFA', 'Units'] = 'g' - stripe_demands.loc['PID', 'Units'] = 'rad' - stripe_demands.insert(1, 'Family', "") - stripe_demands['Family'] = 'lognormal' - stripe_demands.rename(columns={'median': 'Theta_0'}, inplace=True) - stripe_demands.rename(columns={'log_std': 'Theta_1'}, inplace=True) - ndims = stripe_demands.shape[0] - demand_types = stripe_demands.index - perfect_CORR = pd.DataFrame( - np.ones((ndims, ndims)), columns=demand_types, index=demand_types - ) - - PAL = Assessment({"PrintLog": True, "Seed": 415}) - PAL.demand.load_model({'marginals': stripe_demands, 'correlation': perfect_CORR}) - PAL.demand.generate_sample({"SampleSize": sample_size}) - - demand_sample = PAL.demand.save_sample() - - delta_y = 0.0075 - PID = demand_sample['PID'] - RID = PAL.demand.estimate_RID(PID, {'yield_drift': delta_y}) - demand_sample_ext = pd.concat([demand_sample, RID], axis=1) - Sa_vals = [0.158, 0.387, 0.615, 0.843, 1.071, 1.299, 1.528, 1.756] - demand_sample_ext[('SA_1.13', 0, 1)] = Sa_vals[int(stripe) - 1] - demand_sample_ext.T.insert(0, 'Units', "") - demand_sample_ext.loc['Units', ['PFA', 'SA_1.13']] = 'g' - demand_sample_ext.loc['Units', ['PID', 'RID']] = 'rad' - - PAL.demand.load_sample(demand_sample_ext) - - cmp_marginals = pd.read_csv( - 'pelicun/tests/compatibility/PRJ-3411v5/CMP_marginals.csv', index_col=0 - ) - - PAL.stories = 4 - PAL.asset.load_cmp_model({'marginals': cmp_marginals}) - PAL.asset.generate_cmp_sample() - - cmp_sample = PAL.asset.save_cmp_sample() - assert cmp_sample is not None - - with pytest.warns(PelicunWarning): - P58_data = PAL.get_default_data('fragility_DB_FEMA_P58_2nd') - - cmp_list = cmp_marginals.index.unique().values[:-3] - P58_data_for_this_assessment = P58_data.loc[cmp_list, :].sort_values( - 'Incomplete', ascending=False - ) - additional_fragility_db = P58_data_for_this_assessment.sort_index() - - P58_metadata = PAL.get_default_metadata('fragility_DB_FEMA_P58_2nd') - assert P58_metadata is not None - - additional_fragility_db.loc[ - ['D.20.22.013a', 'D.20.22.023a', 'D.20.22.023b'], - [('LS1', 'Theta_1'), ('LS2', 'Theta_1')], - ] = 0.5 - additional_fragility_db.loc['D.20.31.013b', ('LS1', 'Theta_1')] = 0.5 - additional_fragility_db.loc['D.20.61.013b', ('LS1', 'Theta_1')] = 0.5 - additional_fragility_db.loc['D.30.31.013i', ('LS1', 'Theta_0')] = 1.5 # g - additional_fragility_db.loc['D.30.31.013i', ('LS1', 'Theta_1')] = 0.5 - additional_fragility_db.loc['D.30.31.023i', ('LS1', 'Theta_0')] = 1.5 # g - additional_fragility_db.loc['D.30.31.023i', ('LS1', 'Theta_1')] = 0.5 - additional_fragility_db.loc['D.30.52.013i', ('LS1', 'Theta_0')] = 1.5 # g - additional_fragility_db.loc['D.30.52.013i', ('LS1', 'Theta_1')] = 0.5 - additional_fragility_db['Incomplete'] = 0 - - additional_fragility_db.loc[ - 'excessiveRID', - [ - ('Demand', 'Directional'), - ('Demand', 'Offset'), - ('Demand', 'Type'), - ('Demand', 'Unit'), - ], - ] = [1, 0, 'Residual Interstory Drift Ratio', 'rad'] - additional_fragility_db.loc[ - 'excessiveRID', [('LS1', 'Family'), ('LS1', 'Theta_0'), ('LS1', 'Theta_1')] - ] = ['lognormal', 0.01, 0.3] - additional_fragility_db.loc[ - 'irreparable', - [ - ('Demand', 'Directional'), - ('Demand', 'Offset'), - ('Demand', 'Type'), - ('Demand', 'Unit'), - ], - ] = [1, 0, 'Peak Spectral Acceleration|1.13', 'g'] - additional_fragility_db.loc['irreparable', ('LS1', 'Theta_0')] = 1e10 - additional_fragility_db.loc[ - 'collapse', - [ - ('Demand', 'Directional'), - ('Demand', 'Offset'), - ('Demand', 'Type'), - ('Demand', 'Unit'), - ], - ] = [1, 0, 'Peak Spectral Acceleration|1.13', 'g'] - additional_fragility_db.loc[ - 'collapse', [('LS1', 'Family'), ('LS1', 'Theta_0'), ('LS1', 'Theta_1')] - ] = ['lognormal', 1.35, 0.5] - additional_fragility_db['Incomplete'] = 0 - - with pytest.warns(PelicunWarning): - PAL.damage.load_damage_model( - [additional_fragility_db, 'PelicunDefault/fragility_DB_FEMA_P58_2nd.csv'] - ) - - # FEMA P58 uses the following process: - dmg_process = { - "1_collapse": {"DS1": "ALL_NA"}, - "2_excessiveRID": {"DS1": "irreparable_DS1"}, - } - PAL.damage.calculate(dmg_process=dmg_process) - - damage_sample = PAL.damage.save_sample() - assert damage_sample is not None - - drivers = [f'DMG-{cmp}' for cmp in cmp_marginals.index.unique()] - drivers = drivers[:-3] + drivers[-2:] - - loss_models = cmp_marginals.index.unique().tolist()[:-3] - loss_models += ['replacement'] * 2 - loss_map = pd.DataFrame(loss_models, columns=['BldgRepair'], index=drivers) - - with pytest.warns(PelicunWarning): - P58_data = PAL.get_default_data('bldg_repair_DB_FEMA_P58_2nd') - - P58_data_for_this_assessment = P58_data.loc[ - loss_map['BldgRepair'].values[:-2], : - ] - - additional_consequences = pd.DataFrame( - columns=pd.MultiIndex.from_tuples( - [ - ('Incomplete', ''), - ('Quantity', 'Unit'), - ('DV', 'Unit'), - ('DS1', 'Theta_0'), - ] - ), - index=pd.MultiIndex.from_tuples( - [('replacement', 'Cost'), ('replacement', 'Time')] - ), - ) - additional_consequences.loc[('replacement', 'Cost')] = [ - 0, - '1 EA', - 'USD_2011', - 21600000, - ] - additional_consequences.loc[('replacement', 'Time')] = [ - 0, - '1 EA', - 'worker_day', - 12500, - ] - - with pytest.warns(PelicunWarning): - PAL.bldg_repair.load_model( - [ - additional_consequences, - "PelicunDefault/bldg_repair_DB_FEMA_P58_2nd.csv", - ], - loss_map, - ) - - PAL.bldg_repair.calculate() - - with pytest.warns(PelicunWarning): - loss_sample = PAL.bldg_repair.sample - assert loss_sample is not None - - with pytest.warns(PelicunWarning): - agg_DF = PAL.bldg_repair.aggregate_losses() - assert agg_DF is not None diff --git a/pelicun/tests/dl_calculation/__init__.py b/pelicun/tests/dl_calculation/__init__.py new file mode 100644 index 000000000..72c332008 --- /dev/null +++ b/pelicun/tests/dl_calculation/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/tests/dl_calculation/e1/__init__.py b/pelicun/tests/dl_calculation/e1/__init__.py new file mode 100644 index 000000000..72c332008 --- /dev/null +++ b/pelicun/tests/dl_calculation/e1/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/tests/dl_calculation/e1/test_e1.py b/pelicun/tests/dl_calculation/e1/test_e1.py index 46d77380a..1db78a984 100644 --- a/pelicun/tests/dl_calculation/e1/test_e1.py +++ b/pelicun/tests/dl_calculation/e1/test_e1.py @@ -1,30 +1,57 @@ -""" -DL Calculation Example 1 +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . + +"""DL Calculation Example 1.""" -""" - -import tempfile import os import shutil +import tempfile from pathlib import Path -import pytest -from pelicun.warnings import PelicunWarning -from pelicun.tools.DL_calculation import run_pelicun +from typing import Generator +import pytest -# pylint: disable=missing-function-docstring -# pylint: disable=missing-yield-doc -# pylint: disable=missing-yield-type-doc -# pylint: disable=redefined-outer-name +from pelicun.pelicun_warnings import PelicunWarning +from pelicun.tools.DL_calculation import run_pelicun @pytest.fixture -def obtain_temp_dir(): - +def obtain_temp_dir() -> Generator: # get the path of this file this_file = __file__ - initial_dir = os.getcwd() + initial_dir = Path.cwd() this_dir = str(Path(this_file).parent) temp_dir = tempfile.mkdtemp() @@ -36,9 +63,11 @@ def obtain_temp_dir(): os.chdir(initial_dir) -def test_dl_calculation_1(obtain_temp_dir): +def test_dl_calculation_1(obtain_temp_dir: str) -> None: + this_dir: str + temp_dir: str - this_dir, temp_dir = obtain_temp_dir + this_dir, temp_dir = obtain_temp_dir # type: ignore # Copy all input files to a temporary directory. # All outputs will also go there. @@ -55,23 +84,17 @@ def test_dl_calculation_1(obtain_temp_dir): os.chdir(temp_dir) # run - with pytest.warns(PelicunWarning): - return_int = run_pelicun( - demand_file='response.csv', - config_path='8000-AIM.json', - output_path=None, - coupled_EDP=True, - realizations='100', - auto_script_path='PelicunDefault/Hazus_Earthquake_IM.py', - detailed_results=False, - regional=True, - output_format=None, - custom_model_dir=None, - color_warnings=False, - ) - - # Python is written in C after all. - assert return_int == 0 + run_pelicun( + demand_file='response.csv', + config_path='8000-AIM.json', + output_path=None, + coupled_edp=True, + realizations=100, + auto_script_path='PelicunDefault/Hazus_Earthquake_IM.py', + detailed_results=False, + output_format=None, + custom_model_dir=None, + ) # # Test files diff --git a/pelicun/tests/dl_calculation/e1_no_autopop/8000-AIM.json b/pelicun/tests/dl_calculation/e1_no_autopop/8000-AIM.json new file mode 100644 index 000000000..122c6a900 --- /dev/null +++ b/pelicun/tests/dl_calculation/e1_no_autopop/8000-AIM.json @@ -0,0 +1,87 @@ +{ + "GeneralInformation": { + "NumberOfStories": 1, + "YearBuilt": 1900, + "StructureType": "C1", + "OccupancyClass": "EDU1", + "units": { + "force": "kips", + "length": "ft", + "time": "sec" + }, + "DesignLevel": "LC", + "BuildingRise": "L" + }, + "assetType": "Buildings", + "Applications": { + "DL": { + "ApplicationData": { + "ground_failure": false + } + } + }, + "DL": { + "Asset": { + "ComponentAssignmentFile": "CMP_QNT.csv", + "ComponentDatabase": "Hazus Earthquake - Buildings", + "NumberOfStories": "1", + "OccupancyType": "EDU1", + "PlanArea": "1" + }, + "Damage": { + "DamageProcess": "Hazus Earthquake", + "ScalingSpecification": {"LF.C1.L.LC-1-1":{"LS1": "*1.2"}, "collapse-0-1": {"ALL": "*1.2"}} + }, + "Demands": { + "DemandFilePath": "response.csv", + "SampleSize": "100", + "CoupledDemands": true + }, + "Losses": { + "Repair": { + "ConsequenceDatabase": "Hazus Earthquake - Buildings", + "MapApproach": "Automatic" + } + }, + "Options": { + "NonDirectionalMultipliers": { + "ALL": 1.0 + } + }, + "Outputs": { + "Demand": { + "Sample": true, + "Statistics": false + }, + "Asset": { + "Sample": true, + "Statistics": false + }, + "Damage": { + "Sample": false, + "Statistics": false, + "GroupedSample": true, + "GroupedStatistics": true + }, + "Loss": { + "Repair": { + "Sample": true, + "Statistics": true, + "GroupedSample": true, + "GroupedStatistics": false, + "AggregateSample": true, + "AggregateStatistics": true + } + }, + "Format": { + "CSV": false, + "JSON": true + }, + "Settings": { + "CondenseDS": true, + "SimpleIndexInJSON": true, + "AggregateColocatedComponentResults": true + } + } + } +} diff --git a/pelicun/tests/dl_calculation/e1_no_autopop/CMP_QNT.csv b/pelicun/tests/dl_calculation/e1_no_autopop/CMP_QNT.csv new file mode 100644 index 000000000..8ecf8aaef --- /dev/null +++ b/pelicun/tests/dl_calculation/e1_no_autopop/CMP_QNT.csv @@ -0,0 +1,2 @@ +,Units,Location,Direction,Theta_0,Family +LF.C1.L.LC,ea,1,1,1,N/A diff --git a/pelicun/tests/dl_calculation/e1_no_autopop/__init__.py b/pelicun/tests/dl_calculation/e1_no_autopop/__init__.py new file mode 100644 index 000000000..72c332008 --- /dev/null +++ b/pelicun/tests/dl_calculation/e1_no_autopop/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/tests/dl_calculation/e1_no_autopop/response.csv b/pelicun/tests/dl_calculation/e1_no_autopop/response.csv new file mode 100644 index 000000000..3034fb85b --- /dev/null +++ b/pelicun/tests/dl_calculation/e1_no_autopop/response.csv @@ -0,0 +1,101 @@ +,1-PGA-1-1 +0.000000000000000000e+00,6.849029933956130911e+00 +1.000000000000000000e+00,1.237134226051886543e+01 +2.000000000000000000e+00,1.972131621939452018e+01 +3.000000000000000000e+00,9.152152931865703778e+00 +4.000000000000000000e+00,4.762545330142875954e+00 +5.000000000000000000e+00,5.119594082152485903e+00 +6.000000000000000000e+00,8.985259370480688901e+00 +7.000000000000000000e+00,5.800429585653212428e+00 +8.000000000000000000e+00,6.721154036788936637e+00 +9.000000000000000000e+00,1.180442967680789756e+01 +1.000000000000000000e+01,1.757573848172475195e+01 +1.100000000000000000e+01,2.408892253534509109e+01 +1.200000000000000000e+01,5.366956767518109572e+00 +1.300000000000000000e+01,6.689460282908749278e+00 +1.400000000000000000e+01,9.837892431968002782e+00 +1.500000000000000000e+01,3.108201101576595793e+00 +1.600000000000000000e+01,7.969306015954598976e+00 +1.700000000000000000e+01,6.239227951703175457e+00 +1.800000000000000000e+01,1.265503535460110918e+01 +1.900000000000000000e+01,5.152699658386873161e+00 +2.000000000000000000e+01,8.945841984822743953e+00 +2.100000000000000000e+01,5.271650608692700857e+00 +2.200000000000000000e+01,1.217883862654925764e+01 +2.300000000000000000e+01,4.565064448801355645e+00 +2.400000000000000000e+01,2.827185963889101927e+01 +2.500000000000000000e+01,8.360633657235220895e+00 +2.600000000000000000e+01,1.771275449651086475e+01 +2.700000000000000000e+01,1.353470186586244495e+01 +2.800000000000000000e+01,1.585083782737845048e+01 +2.900000000000000000e+01,7.743412977553257193e+00 +3.000000000000000000e+01,1.673262791659537640e+01 +3.100000000000000000e+01,1.125514571474493408e+01 +3.200000000000000000e+01,6.706408356999864928e+00 +3.300000000000000000e+01,7.770377729315534943e+00 +3.400000000000000000e+01,9.665443132282717897e+00 +3.500000000000000000e+01,1.723980753309739811e+01 +3.600000000000000000e+01,1.291221957249968177e+01 +3.700000000000000000e+01,5.012730578330669040e+00 +3.800000000000000000e+01,1.143700619441758626e+01 +3.900000000000000000e+01,1.252350066480562596e+01 +4.000000000000000000e+01,6.383801705492390788e+00 +4.100000000000000000e+01,8.314748267032781470e+00 +4.200000000000000000e+01,8.332817023133479495e+00 +4.300000000000000000e+01,7.344179823126522066e+00 +4.400000000000000000e+01,1.645381636611937282e+01 +4.500000000000000000e+01,1.204983450209533657e+01 +4.600000000000000000e+01,1.165673673862969828e+01 +4.700000000000000000e+01,7.513555832366318299e+00 +4.800000000000000000e+01,2.024437216200684730e+01 +4.900000000000000000e+01,8.920834603836945931e+00 +5.000000000000000000e+01,1.080188057233199572e+01 +5.100000000000000000e+01,6.199736427905365943e+00 +5.200000000000000000e+01,1.372165205481582362e+01 +5.300000000000000000e+01,7.684718998480578378e+00 +5.400000000000000000e+01,5.171043463741079371e+00 +5.500000000000000000e+01,2.111664537027713706e+01 +5.600000000000000000e+01,1.019006581042710913e+01 +5.700000000000000000e+01,4.275116867520939223e+00 +5.800000000000000000e+01,1.899195644078132261e+01 +5.900000000000000000e+01,1.213049449493617438e+01 +6.000000000000000000e+01,4.832690227828255303e+00 +6.100000000000000000e+01,3.791254293305350132e+00 +6.200000000000000000e+01,8.030002531225640894e+00 +6.300000000000000000e+01,1.163245027176597013e+01 +6.400000000000000000e+01,2.171715232778592508e+01 +6.500000000000000000e+01,7.206784171136174422e+00 +6.600000000000000000e+01,4.534544778468278636e+00 +6.700000000000000000e+01,8.180354349576319350e+00 +6.800000000000000000e+01,2.096016461144264653e+01 +6.900000000000000000e+01,1.020904287768493468e+01 +7.000000000000000000e+01,3.578274393687075783e+00 +7.100000000000000000e+01,1.207216137873451700e+01 +7.200000000000000000e+01,1.899808690168648084e+01 +7.300000000000000000e+01,1.447931410453573520e+01 +7.400000000000000000e+01,4.652951138159096445e+00 +7.500000000000000000e+01,9.959855894302648949e+00 +7.600000000000000000e+01,4.533117400993131874e+00 +7.700000000000000000e+01,7.248896994573899910e+00 +7.800000000000000000e+01,8.842184263094427621e+00 +7.900000000000000000e+01,4.215076299759047629e+00 +8.000000000000000000e+01,1.529557773187450032e+01 +8.100000000000000000e+01,4.175349145066248546e+00 +8.200000000000000000e+01,1.009543326050395251e+01 +8.300000000000000000e+01,8.690186504818180779e+00 +8.400000000000000000e+01,1.547032449114289854e+01 +8.500000000000000000e+01,7.042901121410118925e+00 +8.600000000000000000e+01,4.125904243435575935e+00 +8.700000000000000000e+01,1.376223081867616571e+01 +8.800000000000000000e+01,7.388148474784416386e+00 +8.900000000000000000e+01,1.453892207787288093e+01 +9.000000000000000000e+01,1.299062984361380124e+01 +9.100000000000000000e+01,6.391741821717948469e+00 +9.200000000000000000e+01,8.088304936390127153e+00 +9.300000000000000000e+01,2.300799414165177481e+01 +9.400000000000000000e+01,1.575348473932006321e+01 +9.500000000000000000e+01,1.019287352203471109e+01 +9.600000000000000000e+01,1.786715222266329661e+01 +9.700000000000000000e+01,1.328402105666898869e+01 +9.800000000000000000e+01,5.457579231912252915e+00 +9.900000000000000000e+01,5.857004480627420406e+00 diff --git a/pelicun/tests/dl_calculation/e1_no_autopop/test_e1.py b/pelicun/tests/dl_calculation/e1_no_autopop/test_e1.py new file mode 100644 index 000000000..8f823c815 --- /dev/null +++ b/pelicun/tests/dl_calculation/e1_no_autopop/test_e1.py @@ -0,0 +1,134 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . + +"""DL Calculation Example 1.""" + +import os +import shutil +import tempfile +from pathlib import Path +from typing import Generator + +import pytest + +from pelicun.pelicun_warnings import PelicunWarning +from pelicun.tools.DL_calculation import run_pelicun + + +@pytest.fixture +def obtain_temp_dir() -> Generator: + # get the path of this file + this_file = __file__ + + initial_dir = Path.cwd() + this_dir = str(Path(this_file).parent) + + temp_dir = tempfile.mkdtemp() + + yield this_dir, temp_dir + + # go back to the right directory, otherwise any tests that follow + # could have issues. + os.chdir(initial_dir) + + +def test_dl_calculation_1(obtain_temp_dir: str) -> None: + this_dir: str + temp_dir: str + + this_dir, temp_dir = obtain_temp_dir # type: ignore + + # Copy all input files to a temporary directory. + # All outputs will also go there. + # This approach is more robust to changes in the output files over + # time. + + os.chdir(this_dir) + temp_dir = tempfile.mkdtemp() + # copy input files + for file_name in ('8000-AIM.json', 'response.csv', 'CMP_QNT.csv'): + shutil.copy(f'{this_dir}/{file_name}', f'{temp_dir}/{file_name}') + + # change directory to there + os.chdir(temp_dir) + + # run + run_pelicun( + demand_file='response.csv', + config_path='8000-AIM.json', + output_path=None, + coupled_edp=True, + realizations=100, + auto_script_path=None, + detailed_results=False, + output_format=None, + custom_model_dir=None, + ) + + # + # Test files + # + + # Ensure the number of files is as expected + num_files = sum(1 for entry in Path(temp_dir).iterdir() if entry.is_file()) + assert num_files == 18 + + # Verify their names + files = { + '8000-AIM.json', + 'CMP_QNT.csv', + 'CMP_sample.json', + 'DEM_sample.json', + 'DL_summary.csv', + 'DL_summary.json', + 'DL_summary_stats.csv', + 'DL_summary_stats.json', + 'DMG_grp.json', + 'DMG_grp_stats.json', + 'DV_repair_agg.json', + 'DV_repair_agg_stats.json', + 'DV_repair_grp.json', + 'DV_repair_sample.json', + 'DV_repair_stats.json', + 'pelicun_log.txt', + 'pelicun_log_warnings.txt', + 'response.csv', + } + + for file in files: + assert Path(f'{temp_dir}/{file}').is_file() + + # + # Check the values: TODO + # diff --git a/pelicun/tests/dl_calculation/e2/__init__.py b/pelicun/tests/dl_calculation/e2/__init__.py new file mode 100644 index 000000000..72c332008 --- /dev/null +++ b/pelicun/tests/dl_calculation/e2/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/tests/dl_calculation/e2/test_e2.py b/pelicun/tests/dl_calculation/e2/test_e2.py index d822f406c..d9ed638e5 100644 --- a/pelicun/tests/dl_calculation/e2/test_e2.py +++ b/pelicun/tests/dl_calculation/e2/test_e2.py @@ -1,30 +1,59 @@ -""" -DL Calculation Example 2 +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . + +"""DL Calculation Example 2.""" + +from __future__ import annotations -""" - -import tempfile import os import shutil +import tempfile from pathlib import Path -import pytest -from pelicun.warnings import PelicunWarning -from pelicun.tools.DL_calculation import run_pelicun +from typing import Generator +import pytest -# pylint: disable=missing-function-docstring -# pylint: disable=missing-yield-doc -# pylint: disable=missing-yield-type-doc -# pylint: disable=redefined-outer-name +from pelicun.pelicun_warnings import PelicunWarning +from pelicun.tools.DL_calculation import run_pelicun @pytest.fixture -def obtain_temp_dir(): - +def obtain_temp_dir() -> Generator: # get the path of this file this_file = __file__ - initial_dir = os.getcwd() + initial_dir = Path.cwd() this_dir = str(Path(this_file).parent) temp_dir = tempfile.mkdtemp() @@ -36,8 +65,7 @@ def obtain_temp_dir(): os.chdir(initial_dir) -def test_dl_calculation_2(obtain_temp_dir): - +def test_dl_calculation_2(obtain_temp_dir: tuple[str, str]) -> None: this_dir, temp_dir = obtain_temp_dir # Copy all input files to a temporary directory. @@ -52,22 +80,17 @@ def test_dl_calculation_2(obtain_temp_dir): os.chdir(temp_dir) # run - with pytest.warns(PelicunWarning): - return_int = run_pelicun( - demand_file='response.csv', - config_path='1-AIM.json', - output_path=None, - coupled_EDP=True, - realizations='100', - auto_script_path='PelicunDefault/Hazus_Earthquake_Story.py', - detailed_results=False, - regional=True, - output_format=None, - custom_model_dir=None, - color_warnings=False, - ) - - assert return_int == 0 + run_pelicun( + demand_file='response.csv', + config_path='1-AIM.json', + output_path=None, + coupled_edp=True, + realizations=100, + auto_script_path='PelicunDefault/Hazus_Earthquake_Story.py', + detailed_results=False, + output_format=None, + custom_model_dir=None, + ) # # Test files diff --git a/pelicun/tests/dl_calculation/e3/__init__.py b/pelicun/tests/dl_calculation/e3/__init__.py new file mode 100644 index 000000000..72c332008 --- /dev/null +++ b/pelicun/tests/dl_calculation/e3/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/tests/dl_calculation/e3/test_e3.py b/pelicun/tests/dl_calculation/e3/test_e3.py index 34b5c1730..11745c4d4 100644 --- a/pelicun/tests/dl_calculation/e3/test_e3.py +++ b/pelicun/tests/dl_calculation/e3/test_e3.py @@ -1,30 +1,59 @@ -""" -DL Calculation Example 3 +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . + +"""DL Calculation Example 3.""" + +from __future__ import annotations -""" - -import tempfile import os import shutil +import tempfile from pathlib import Path -import pytest -from pelicun.warnings import PelicunWarning -from pelicun.tools.DL_calculation import run_pelicun +from typing import Generator +import pytest -# pylint: disable=missing-function-docstring -# pylint: disable=missing-yield-doc -# pylint: disable=missing-yield-type-doc -# pylint: disable=redefined-outer-name +from pelicun.pelicun_warnings import PelicunWarning +from pelicun.tools.DL_calculation import run_pelicun @pytest.fixture -def obtain_temp_dir(): - +def obtain_temp_dir() -> Generator: # get the path of this file this_file = __file__ - initial_dir = os.getcwd() + initial_dir = Path.cwd() this_dir = str(Path(this_file).parent) temp_dir = tempfile.mkdtemp() @@ -36,8 +65,7 @@ def obtain_temp_dir(): os.chdir(initial_dir) -def test_dl_calculation_3(obtain_temp_dir): - +def test_dl_calculation_3(obtain_temp_dir: tuple[str, str]) -> None: this_dir, temp_dir = obtain_temp_dir # Copy all input files to a temporary directory. @@ -50,22 +78,17 @@ def test_dl_calculation_3(obtain_temp_dir): os.chdir(temp_dir) # run - with pytest.warns(PelicunWarning): - return_int = run_pelicun( - demand_file='response.csv', - config_path='170-AIM.json', - output_path=None, - coupled_EDP=False, - realizations='100', - auto_script_path='PelicunDefault/Hazus_Earthquake_Story.py', - detailed_results=False, - regional=True, - output_format=None, - custom_model_dir=None, - color_warnings=False, - ) - - assert return_int == 0 + run_pelicun( + demand_file='response.csv', + config_path='170-AIM.json', + output_path=None, + coupled_edp=False, + realizations=100, + auto_script_path='PelicunDefault/Hazus_Earthquake_Story.py', + detailed_results=False, + output_format=None, + custom_model_dir=None, + ) # # Test files diff --git a/pelicun/tests/dl_calculation/e4/__init__.py b/pelicun/tests/dl_calculation/e4/__init__.py new file mode 100644 index 000000000..72c332008 --- /dev/null +++ b/pelicun/tests/dl_calculation/e4/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/tests/dl_calculation/e4/test_e4.py b/pelicun/tests/dl_calculation/e4/test_e4.py index 8784d8524..45544beeb 100644 --- a/pelicun/tests/dl_calculation/e4/test_e4.py +++ b/pelicun/tests/dl_calculation/e4/test_e4.py @@ -1,30 +1,59 @@ -""" -DL Calculation Example 4 +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . + +"""DL Calculation Example 4.""" + +from __future__ import annotations -""" - -import tempfile import os import shutil +import tempfile from pathlib import Path -import pytest -from pelicun.warnings import PelicunWarning -from pelicun.tools.DL_calculation import run_pelicun +from typing import Generator +import pytest -# pylint: disable=missing-function-docstring -# pylint: disable=missing-yield-doc -# pylint: disable=missing-yield-type-doc -# pylint: disable=redefined-outer-name +from pelicun.pelicun_warnings import PelicunWarning +from pelicun.tools.DL_calculation import run_pelicun @pytest.fixture -def obtain_temp_dir(): - +def obtain_temp_dir() -> Generator: # get the path of this file this_file = __file__ - initial_dir = os.getcwd() + initial_dir = Path.cwd() this_dir = str(Path(this_file).parent) temp_dir = tempfile.mkdtemp() @@ -36,8 +65,7 @@ def obtain_temp_dir(): os.chdir(initial_dir) -def test_dl_calculation_4(obtain_temp_dir): - +def test_dl_calculation_4(obtain_temp_dir: tuple[str, str]) -> None: this_dir, temp_dir = obtain_temp_dir # Copy all input files to a temporary directory. @@ -55,22 +83,17 @@ def test_dl_calculation_4(obtain_temp_dir): os.chdir(temp_dir) # run - with pytest.warns(PelicunWarning): - return_int = run_pelicun( - demand_file='response.csv', - config_path='0-AIM.json', - output_path=None, - coupled_EDP=True, - realizations='100', - auto_script_path='PelicunDefault/Hazus_Earthquake_Story.py', - detailed_results=False, - regional=True, - output_format=None, - custom_model_dir=None, - color_warnings=False, - ) - - assert return_int == 0 + run_pelicun( + demand_file='response.csv', + config_path='0-AIM.json', + output_path=None, + coupled_edp=True, + realizations=100, + auto_script_path='PelicunDefault/Hazus_Earthquake_Story.py', + detailed_results=False, + output_format=None, + custom_model_dir=None, + ) # # Test files diff --git a/pelicun/tests/dl_calculation/e5/__init__.py b/pelicun/tests/dl_calculation/e5/__init__.py new file mode 100644 index 000000000..72c332008 --- /dev/null +++ b/pelicun/tests/dl_calculation/e5/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/tests/dl_calculation/e5/test_e5.py b/pelicun/tests/dl_calculation/e5/test_e5.py index 82fcd8493..451afb654 100644 --- a/pelicun/tests/dl_calculation/e5/test_e5.py +++ b/pelicun/tests/dl_calculation/e5/test_e5.py @@ -1,30 +1,59 @@ -""" -DL Calculation Example 5 +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . + +"""DL Calculation Example 5.""" + +from __future__ import annotations -""" - -import tempfile import os import shutil +import tempfile from pathlib import Path -import pytest -from pelicun.warnings import PelicunWarning -from pelicun.tools.DL_calculation import run_pelicun +from typing import Generator +import pytest -# pylint: disable=missing-function-docstring -# pylint: disable=missing-yield-doc -# pylint: disable=missing-yield-type-doc -# pylint: disable=redefined-outer-name +from pelicun.pelicun_warnings import PelicunWarning +from pelicun.tools.DL_calculation import run_pelicun @pytest.fixture -def obtain_temp_dir(): - +def obtain_temp_dir() -> Generator: # get the path of this file this_file = __file__ - initial_dir = os.getcwd() + initial_dir = Path.cwd() this_dir = str(Path(this_file).parent) temp_dir = tempfile.mkdtemp() @@ -36,8 +65,7 @@ def obtain_temp_dir(): os.chdir(initial_dir) -def test_dl_calculation_5(obtain_temp_dir): - +def test_dl_calculation_5(obtain_temp_dir: tuple[str, str]) -> None: this_dir, temp_dir = obtain_temp_dir # Copy all input files to a temporary directory. @@ -55,22 +83,17 @@ def test_dl_calculation_5(obtain_temp_dir): os.chdir(temp_dir) # run - with pytest.warns(PelicunWarning): - return_int = run_pelicun( - demand_file='response.csv', - config_path='1-AIM.json', - output_path=None, - coupled_EDP=True, - realizations='100', - auto_script_path='PelicunDefault/Hazus_Earthquake_IM.py', - detailed_results=False, - regional=True, - output_format=None, - custom_model_dir=None, - color_warnings=False, - ) - - assert return_int == 0 + run_pelicun( + demand_file='response.csv', + config_path='1-AIM.json', + output_path=None, + coupled_edp=True, + realizations=100, + auto_script_path='PelicunDefault/Hazus_Earthquake_IM.py', + detailed_results=False, + output_format=None, + custom_model_dir=None, + ) # # Test files diff --git a/pelicun/tests/dl_calculation/e6/__init__.py b/pelicun/tests/dl_calculation/e6/__init__.py new file mode 100644 index 000000000..72c332008 --- /dev/null +++ b/pelicun/tests/dl_calculation/e6/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/tests/dl_calculation/e6/test_e6.py b/pelicun/tests/dl_calculation/e6/test_e6.py index 803dfbb7e..5f6c9cf71 100644 --- a/pelicun/tests/dl_calculation/e6/test_e6.py +++ b/pelicun/tests/dl_calculation/e6/test_e6.py @@ -1,30 +1,59 @@ -""" -DL Calculation Example 6 +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . + +"""DL Calculation Example 6.""" + +from __future__ import annotations -""" - -import tempfile import os import shutil +import tempfile from pathlib import Path -import pytest -from pelicun.warnings import PelicunWarning -from pelicun.tools.DL_calculation import run_pelicun +from typing import Generator +import pytest -# pylint: disable=missing-function-docstring -# pylint: disable=missing-yield-doc -# pylint: disable=missing-yield-type-doc -# pylint: disable=redefined-outer-name +from pelicun.pelicun_warnings import PelicunWarning +from pelicun.tools.DL_calculation import run_pelicun @pytest.fixture -def obtain_temp_dir(): - +def obtain_temp_dir() -> Generator: # get the path of this file this_file = __file__ - initial_dir = os.getcwd() + initial_dir = Path.cwd() this_dir = str(Path(this_file).parent) temp_dir = tempfile.mkdtemp() @@ -36,8 +65,7 @@ def obtain_temp_dir(): os.chdir(initial_dir) -def test_dl_calculation_6(obtain_temp_dir): - +def test_dl_calculation_6(obtain_temp_dir: tuple[str, str]) -> None: this_dir, temp_dir = obtain_temp_dir # Copy all input files to a temporary directory. @@ -55,22 +83,17 @@ def test_dl_calculation_6(obtain_temp_dir): os.chdir(temp_dir) # run - with pytest.warns(PelicunWarning): - return_int = run_pelicun( - demand_file='response.csv', - config_path='1-AIM.json', - output_path=None, - coupled_EDP=True, - realizations='100', - auto_script_path='PelicunDefault/Hazus_Earthquake_IM.py', - detailed_results=False, - regional=True, - output_format=None, - custom_model_dir=None, - color_warnings=False, - ) - - assert return_int == 0 + run_pelicun( + demand_file='response.csv', + config_path='1-AIM.json', + output_path=None, + coupled_edp=True, + realizations=100, + auto_script_path='PelicunDefault/Hazus_Earthquake_IM.py', + detailed_results=False, + output_format=None, + custom_model_dir=None, + ) # # Test files diff --git a/pelicun/tests/dl_calculation/e7/__init__.py b/pelicun/tests/dl_calculation/e7/__init__.py new file mode 100644 index 000000000..72c332008 --- /dev/null +++ b/pelicun/tests/dl_calculation/e7/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/tests/dl_calculation/e7/auto_HU_NJ.py b/pelicun/tests/dl_calculation/e7/auto_HU_NJ.py index 3bd24ffb8..72004d445 100644 --- a/pelicun/tests/dl_calculation/e7/auto_HU_NJ.py +++ b/pelicun/tests/dl_calculation/e7/auto_HU_NJ.py @@ -46,27 +46,29 @@ import pandas as pd -from WindMetaVarRulesets import parse_BIM -from BuildingClassRulesets import building_class -from FloodAssmRulesets import Assm_config -from FloodClassRulesets import FL_config -from WindCECBRulesets import CECB_config -from WindCERBRulesets import CERB_config -from WindMECBRulesets import MECB_config -from WindMERBRulesets import MERB_config -from WindMHRulesets import MH_config -from WindMLRIRulesets import MLRI_config -from WindMLRMRulesets import MLRM_config -from WindMMUHRulesets import MMUH_config -from WindMSFRulesets import MSF_config -from WindSECBRulesets import SECB_config -from WindSERBRulesets import SERB_config -from WindSPMBRulesets import SPMB_config -from WindWMUHRulesets import WMUH_config -from WindWSFRulesets import WSF_config - - -def auto_populate(AIM): +from pelicun.tests.dl_calculation.rulesets.WindMetaVarRulesets import parse_BIM +from pelicun.tests.dl_calculation.rulesets.BuildingClassRulesets import ( + building_class, +) +from pelicun.tests.dl_calculation.rulesets.FloodAssmRulesets import Assm_config +from pelicun.tests.dl_calculation.rulesets.FloodClassRulesets import FL_config +from pelicun.tests.dl_calculation.rulesets.WindCECBRulesets import CECB_config +from pelicun.tests.dl_calculation.rulesets.WindCERBRulesets import CERB_config +from pelicun.tests.dl_calculation.rulesets.WindMECBRulesets import MECB_config +from pelicun.tests.dl_calculation.rulesets.WindMERBRulesets import MERB_config +from pelicun.tests.dl_calculation.rulesets.WindMHRulesets import MH_config +from pelicun.tests.dl_calculation.rulesets.WindMLRIRulesets import MLRI_config +from pelicun.tests.dl_calculation.rulesets.WindMLRMRulesets import MLRM_config +from pelicun.tests.dl_calculation.rulesets.WindMMUHRulesets import MMUH_config +from pelicun.tests.dl_calculation.rulesets.WindMSFRulesets import MSF_config +from pelicun.tests.dl_calculation.rulesets.WindSECBRulesets import SECB_config +from pelicun.tests.dl_calculation.rulesets.WindSERBRulesets import SERB_config +from pelicun.tests.dl_calculation.rulesets.WindSPMBRulesets import SPMB_config +from pelicun.tests.dl_calculation.rulesets.WindWMUHRulesets import WMUH_config +from pelicun.tests.dl_calculation.rulesets.WindWSFRulesets import WSF_config + + +def auto_populate(aim): """ Populates the DL model for hurricane assessments in Atlantic County, NJ @@ -78,7 +80,7 @@ def auto_populate(AIM): Parameters ---------- - AIM_in: dictionary + aim: dictionary Contains the information that is available about the asset and will be used to auto-popualate the damage and loss model. @@ -91,10 +93,10 @@ def auto_populate(AIM): """ # extract the General Information - GI = AIM.get('GeneralInformation', None) + GI = aim.get('GeneralInformation', None) # parse the GI data - GI_ap = parse_BIM(GI, location="NJ", hazards=['wind', 'inundation']) + GI_ap = parse_BIM(GI, location='NJ', hazards=['wind', 'inundation']) # identify the building class bldg_class = building_class(GI_ap, hazard='wind') @@ -130,8 +132,8 @@ def auto_populate(AIM): bldg_config = MH_config(GI_ap) else: raise ValueError( - f"Building class {bldg_class} not recognized by the " - f"auto-population routine." + f'Building class {bldg_class} not recognized by the ' + f'auto-population routine.' ) # prepare the flood rulesets @@ -150,24 +152,24 @@ def auto_populate(AIM): ).T DL_ap = { - "Asset": { - "ComponentAssignmentFile": "CMP_QNT.csv", - "ComponentDatabase": "Hazus Hurricane", - "NumberOfStories": f"{GI_ap['NumberOfStories']}", - "OccupancyType": f"{GI_ap['OccupancyClass']}", - "PlanArea": f"{GI_ap['PlanArea']}", + 'Asset': { + 'ComponentAssignmentFile': 'CMP_QNT.csv', + 'ComponentDatabase': 'Hazus Hurricane', + 'NumberOfStories': f"{GI_ap['NumberOfStories']}", + 'OccupancyType': f"{GI_ap['OccupancyClass']}", + 'PlanArea': f"{GI_ap['PlanArea']}", }, - "Damage": {"DamageProcess": "Hazus Hurricane"}, - "Demands": {}, - "Losses": { - "BldgRepair": { - "ConsequenceDatabase": "Hazus Hurricane", - "MapApproach": "Automatic", - "DecisionVariables": { - "Cost": True, - "Carbon": False, - "Energy": False, - "Time": False, + 'Damage': {'DamageProcess': 'Hazus Hurricane'}, + 'Demands': {}, + 'Losses': { + 'BldgRepair': { + 'ConsequenceDatabase': 'Hazus Hurricane', + 'MapApproach': 'Automatic', + 'DecisionVariables': { + 'Cost': True, + 'Carbon': False, + 'Energy': False, + 'Time': False, }, } }, diff --git a/pelicun/tests/dl_calculation/e7/test_e7.py b/pelicun/tests/dl_calculation/e7/test_e7.py index cdcd1ec2d..9f16c2697 100644 --- a/pelicun/tests/dl_calculation/e7/test_e7.py +++ b/pelicun/tests/dl_calculation/e7/test_e7.py @@ -1,32 +1,60 @@ -""" -DL Calculation Example 7 +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . + +"""DL Calculation Example 7.""" + +from __future__ import annotations -""" - -import tempfile import os import shutil -from glob import glob +import tempfile from pathlib import Path +from typing import Generator + import pytest + # import pandas as pd -from pelicun.warnings import PelicunWarning +from pelicun.pelicun_warnings import PelicunWarning from pelicun.tools.DL_calculation import run_pelicun -# pylint: disable=missing-function-docstring -# pylint: disable=missing-yield-doc -# pylint: disable=missing-yield-type-doc -# pylint: disable=redefined-outer-name - - @pytest.fixture -def obtain_temp_dir(): - +def obtain_temp_dir() -> Generator: # get the path of this file this_file = __file__ - initial_dir = os.getcwd() + initial_dir = Path.cwd() this_dir = str(Path(this_file).parent) temp_dir = tempfile.mkdtemp() @@ -38,8 +66,7 @@ def obtain_temp_dir(): os.chdir(initial_dir) -def test_dl_calculation_7(obtain_temp_dir): - +def test_dl_calculation_7(obtain_temp_dir: tuple[str, str]) -> None: this_dir, temp_dir = obtain_temp_dir # Copy all input files to a temporary directory. @@ -48,8 +75,10 @@ def test_dl_calculation_7(obtain_temp_dir): # time. ruleset_files = [ - Path(x).resolve() - for x in glob('pelicun/tests/dl_calculation/rulesets/*Rulesets.py') + path.resolve() + for path in Path('pelicun/tests/dl_calculation/rulesets').glob( + '*Rulesets.py' + ) ] os.chdir(this_dir) @@ -66,27 +95,22 @@ def test_dl_calculation_7(obtain_temp_dir): os.chdir(temp_dir) # run - with pytest.warns(PelicunWarning): - return_int = run_pelicun( - demand_file='response.csv', - config_path='1-AIM.json', - output_path=None, - coupled_EDP=True, - realizations='100', - auto_script_path='auto_HU_NJ.py', - detailed_results=False, - regional=True, - output_format=None, - custom_model_dir=None, - color_warnings=False, - ) - - assert return_int == 0 + run_pelicun( + demand_file='response.csv', + config_path='1-AIM.json', + output_path=None, + coupled_edp=True, + realizations=100, + auto_script_path='auto_HU_NJ.py', + detailed_results=False, + output_format=None, + custom_model_dir=None, + ) # now remove the ruleset files and auto script for file_path in ruleset_files: - os.remove(f'{temp_dir}/{file_path.name}') - os.remove('auto_HU_NJ.py') + Path(f'{temp_dir}/{file_path.name}').unlink() + Path('auto_HU_NJ.py').unlink() # # Test files diff --git a/pelicun/tests/dl_calculation/e8/__init__.py b/pelicun/tests/dl_calculation/e8/__init__.py new file mode 100644 index 000000000..72c332008 --- /dev/null +++ b/pelicun/tests/dl_calculation/e8/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/tests/dl_calculation/e8/auto_HU_LA.py b/pelicun/tests/dl_calculation/e8/auto_HU_LA.py index d1dd3d048..7402d4c7e 100644 --- a/pelicun/tests/dl_calculation/e8/auto_HU_LA.py +++ b/pelicun/tests/dl_calculation/e8/auto_HU_LA.py @@ -46,13 +46,13 @@ import pandas as pd -from MetaVarRulesets import parse_BIM -from BldgClassRulesets import building_class -from WindWSFRulesets import WSF_config -from WindWMUHRulesets import WMUH_config +from pelicun.tests.dl_calculation.rulesets.MetaVarRulesets import parse_BIM +from pelicun.tests.dl_calculation.rulesets.BldgClassRulesets import building_class +from pelicun.tests.dl_calculation.rulesets.WindWSFRulesets import WSF_config +from pelicun.tests.dl_calculation.rulesets.WindWMUHRulesets import WMUH_config -def auto_populate(AIM): +def auto_populate(aim): """ Populates the DL model for hurricane assessments in Atlantic County, NJ @@ -64,7 +64,7 @@ def auto_populate(AIM): Parameters ---------- - AIM: dictionary + aim: dictionary Contains the information that is available about the asset and will be used to auto-popualate the damage and loss model. @@ -77,12 +77,12 @@ def auto_populate(AIM): """ # extract the General Information - GI = AIM.get('GeneralInformation', None) + GI = aim.get('GeneralInformation', None) # parse the GI data GI_ap = parse_BIM( GI, - location="LA", + location='LA', hazards=[ 'wind', ], @@ -99,8 +99,8 @@ def auto_populate(AIM): bldg_config = WMUH_config(GI_ap) else: raise ValueError( - f"Building class {bldg_class} not recognized by the " - f"auto-population routine." + f'Building class {bldg_class} not recognized by the ' + f'auto-population routine.' ) # drop keys of internal variables from GI_ap dict @@ -118,24 +118,24 @@ def auto_populate(AIM): ).T DL_ap = { - "Asset": { - "ComponentAssignmentFile": "CMP_QNT.csv", - "ComponentDatabase": "Hazus Hurricane", - "NumberOfStories": f"{GI_ap['NumberOfStories']}", - "OccupancyType": f"{GI_ap['OccupancyClass']}", - "PlanArea": f"{GI_ap['PlanArea']}", + 'Asset': { + 'ComponentAssignmentFile': 'CMP_QNT.csv', + 'ComponentDatabase': 'Hazus Hurricane', + 'NumberOfStories': f"{GI_ap['NumberOfStories']}", + 'OccupancyType': f"{GI_ap['OccupancyClass']}", + 'PlanArea': f"{GI_ap['PlanArea']}", }, - "Damage": {"DamageProcess": "Hazus Hurricane"}, - "Demands": {}, - "Losses": { - "BldgRepair": { - "ConsequenceDatabase": "Hazus Hurricane", - "MapApproach": "Automatic", - "DecisionVariables": { - "Cost": True, - "Carbon": False, - "Energy": False, - "Time": False, + 'Damage': {'DamageProcess': 'Hazus Hurricane'}, + 'Demands': {}, + 'Losses': { + 'BldgRepair': { + 'ConsequenceDatabase': 'Hazus Hurricane', + 'MapApproach': 'Automatic', + 'DecisionVariables': { + 'Cost': True, + 'Carbon': False, + 'Energy': False, + 'Time': False, }, } }, diff --git a/pelicun/tests/dl_calculation/e8/test_e8.py b/pelicun/tests/dl_calculation/e8/test_e8.py index 71035e510..040f13c4d 100644 --- a/pelicun/tests/dl_calculation/e8/test_e8.py +++ b/pelicun/tests/dl_calculation/e8/test_e8.py @@ -1,31 +1,59 @@ -""" -DL Calculation Example 8 +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . + +"""DL Calculation Example 8.""" + +from __future__ import annotations -""" - -import tempfile import os import shutil -from glob import glob +import tempfile from pathlib import Path -import pytest -from pelicun.warnings import PelicunWarning -from pelicun.tools.DL_calculation import run_pelicun +from typing import Generator +import pytest -# pylint: disable=missing-function-docstring -# pylint: disable=missing-yield-doc -# pylint: disable=missing-yield-type-doc -# pylint: disable=redefined-outer-name +from pelicun.pelicun_warnings import PelicunWarning +from pelicun.tools.DL_calculation import run_pelicun @pytest.fixture -def obtain_temp_dir(): - +def obtain_temp_dir() -> Generator: # get the path of this file this_file = __file__ - initial_dir = os.getcwd() + initial_dir = Path.cwd() this_dir = str(Path(this_file).parent) temp_dir = tempfile.mkdtemp() @@ -37,8 +65,7 @@ def obtain_temp_dir(): os.chdir(initial_dir) -def test_dl_calculation_8(obtain_temp_dir): - +def test_dl_calculation_8(obtain_temp_dir: tuple[str, str]) -> None: this_dir, temp_dir = obtain_temp_dir # Copy all input files to a temporary directory. @@ -47,8 +74,10 @@ def test_dl_calculation_8(obtain_temp_dir): # time. ruleset_files = [ - Path(x).resolve() - for x in glob('pelicun/tests/dl_calculation/rulesets/*Rulesets.py') + path.resolve() + for path in Path('pelicun/tests/dl_calculation/rulesets').glob( + '*Rulesets.py' + ) ] os.chdir(this_dir) @@ -65,27 +94,22 @@ def test_dl_calculation_8(obtain_temp_dir): os.chdir(temp_dir) # run - with pytest.warns(PelicunWarning): - return_int = run_pelicun( - demand_file='response.csv', - config_path='1-AIM.json', - output_path=None, - coupled_EDP=True, - realizations='100', - auto_script_path='auto_HU_LA.py', - detailed_results=False, - regional=True, - output_format=None, - custom_model_dir=None, - color_warnings=False, - ) - - assert return_int == 0 + run_pelicun( + demand_file='response.csv', + config_path='1-AIM.json', + output_path=None, + coupled_edp=True, + realizations=100, + auto_script_path='auto_HU_LA.py', + detailed_results=False, + output_format=None, + custom_model_dir=None, + ) # now remove the ruleset files and auto script for file_path in ruleset_files: - os.remove(f'{temp_dir}/{file_path.name}') - os.remove('auto_HU_LA.py') + Path(f'{temp_dir}/{file_path.name}').unlink() + Path('auto_HU_LA.py').unlink() # # Test files diff --git a/pelicun/tests/dl_calculation/e9/CustomDLModels/loss_map.csv b/pelicun/tests/dl_calculation/e9/CustomDLModels/loss_map.csv index 1d0c25694..4a585fe4f 100644 --- a/pelicun/tests/dl_calculation/e9/CustomDLModels/loss_map.csv +++ b/pelicun/tests/dl_calculation/e9/CustomDLModels/loss_map.csv @@ -1,4 +1,4 @@ -,BldgRepair -DMG-building.1,generic -DMG-building.2,generic -DMG-building.3andAbove,generic \ No newline at end of file +,Repair +building.1,generic +building.2,generic +building.3andAbove,generic diff --git a/pelicun/tests/dl_calculation/e9/__init__.py b/pelicun/tests/dl_calculation/e9/__init__.py new file mode 100644 index 000000000..72c332008 --- /dev/null +++ b/pelicun/tests/dl_calculation/e9/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/tests/dl_calculation/e9/custom_pop.py b/pelicun/tests/dl_calculation/e9/custom_pop.py index 40b8c3c85..1319a8b7e 100644 --- a/pelicun/tests/dl_calculation/e9/custom_pop.py +++ b/pelicun/tests/dl_calculation/e9/custom_pop.py @@ -8,20 +8,19 @@ import pandas as pd -def auto_populate(AIM): +def auto_populate(aim): """ Populates the DL model for tsunami example using custom fragility functions - Assumptions - ----------- + Assumptions: * Everything relevant to auto-population is provided in the - Buiding Information Model (AIM). + Buiding Information Model (AIM). * The information expected in the AIM file is described in the - parse_AIM method. + parse_AIM method. Parameters ---------- - AIM: dictionary + aim: dictionary Contains the information that is available about the asset and will be used to auto-populate the damage and loss model. @@ -34,10 +33,10 @@ def auto_populate(AIM): """ # parse the AIM data - # print(AIM) # Look in the AIM.json file to see what you can access here + # print(aim) # Look in the AIM.json file to see what you can access here # extract the General Information - GI = AIM.get('GeneralInformation', None) + GI = aim.get('GeneralInformation', None) # GI_ap is the 'extended AIM data - this case no extended AIM data GI_ap = GI.copy() @@ -80,7 +79,7 @@ def auto_populate(AIM): "Damage": {"DamageProcess": "None"}, "Demands": {}, "Losses": { - "BldgRepair": { + "Repair": { "ConsequenceDatabase": "None", "ConsequenceDatabasePath": ( "CustomDLDataFolder/loss_repair_Tsunami.csv" diff --git a/pelicun/tests/dl_calculation/e9/test_e9.py b/pelicun/tests/dl_calculation/e9/test_e9.py index 527ae304b..3cf5a5186 100644 --- a/pelicun/tests/dl_calculation/e9/test_e9.py +++ b/pelicun/tests/dl_calculation/e9/test_e9.py @@ -1,31 +1,59 @@ -""" -DL Calculation Example 9 +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . + +"""DL Calculation Example 9.""" + +from __future__ import annotations -""" - -import tempfile import os import shutil -from glob import glob +import tempfile from pathlib import Path -import pytest -from pelicun.warnings import PelicunWarning -from pelicun.tools.DL_calculation import run_pelicun +from typing import Generator +import pytest -# pylint: disable=missing-function-docstring -# pylint: disable=missing-yield-doc -# pylint: disable=missing-yield-type-doc -# pylint: disable=redefined-outer-name +from pelicun.pelicun_warnings import PelicunWarning +from pelicun.tools.DL_calculation import run_pelicun @pytest.fixture -def obtain_temp_dir(): - +def obtain_temp_dir() -> Generator: # get the path of this file this_file = __file__ - initial_dir = os.getcwd() + initial_dir = Path.cwd() this_dir = str(Path(this_file).parent) temp_dir = tempfile.mkdtemp() @@ -37,8 +65,7 @@ def obtain_temp_dir(): os.chdir(initial_dir) -def test_dl_calculation_9(obtain_temp_dir): - +def test_dl_calculation_9(obtain_temp_dir: tuple[str, str]) -> None: this_dir, temp_dir = obtain_temp_dir # Copy all input files to a temporary directory. @@ -46,9 +73,12 @@ def test_dl_calculation_9(obtain_temp_dir): # This approach is more robust to changes in the output files over # time. ruleset_files = [ - Path(x).resolve() - for x in glob('pelicun/tests/dl_calculation/rulesets/*Rulesets.py') + path.resolve() + for path in Path('pelicun/tests/dl_calculation/rulesets').glob( + '*Rulesets.py' + ) ] + dl_models_dir = Path(f'{this_dir}/CustomDLModels').resolve() os.chdir(this_dir) temp_dir = tempfile.mkdtemp() @@ -64,27 +94,22 @@ def test_dl_calculation_9(obtain_temp_dir): os.chdir(temp_dir) # run - with pytest.warns(PelicunWarning): - return_int = run_pelicun( - demand_file='response.csv', - config_path='3500-AIM.json', - output_path=None, - coupled_EDP=True, - realizations='100', - auto_script_path='custom_pop.py', - detailed_results=False, - regional=True, - output_format=None, - custom_model_dir='./CustomDLModels', - color_warnings=False, - ) - - assert return_int == 0 + run_pelicun( + demand_file='response.csv', + config_path='3500-AIM.json', + output_path=None, + coupled_edp=True, + realizations=100, + auto_script_path='custom_pop.py', + detailed_results=False, + output_format=None, + custom_model_dir='./CustomDLModels', + ) # now remove the ruleset files and auto script for file_path in ruleset_files: - os.remove(f'{temp_dir}/{file_path.name}') - os.remove('custom_pop.py') + Path(f'{temp_dir}/{file_path.name}').unlink() + Path('custom_pop.py').unlink() # # Test files diff --git a/pelicun/tests/dl_calculation/rulesets/BldgClassRulesets.py b/pelicun/tests/dl_calculation/rulesets/BldgClassRulesets.py index 70c6395d8..60432ff41 100644 --- a/pelicun/tests/dl_calculation/rulesets/BldgClassRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/BldgClassRulesets.py @@ -43,6 +43,9 @@ # Meredith Lockhead # Tracy Kijewski-Correa +import random +import numpy as np +import datetime def building_class(BIM, hazard): """ @@ -69,21 +72,14 @@ def building_class(BIM, hazard): if hazard == 'wind': if BIM['BuildingType'] == "Wood": - if (BIM['OccupancyClass'] == 'RES1') or ( - (BIM['RoofShape'] != 'flt') and (BIM['OccupancyClass'] == '') - ): + if ((BIM['OccupancyClass'] == 'RES1') or + ((BIM['RoofShape'] != 'flt') and (BIM['OccupancyClass'] == ''))): # OccupancyClass = RES1 # Wood Single-Family Homes (WSF1 or WSF2) # OR roof type = flat (HAZUS can only map flat to WSF1) # OR default (by '') - if ( - BIM['RoofShape'] == 'flt' - ): # checking if there is a misclassication - BIM['RoofShape'] = ( - # ensure the WSF has gab (by default, note gab - # is more vulneable than hip) - 'gab' - ) + if BIM['RoofShape'] == 'flt': # checking if there is a misclassication + BIM['RoofShape'] = 'gab' # ensure the WSF has gab (by default, note gab is more vulneable than hip) bldg_class = 'WSF' else: # OccupancyClass = RES3, RES5, RES6, or COM8 @@ -91,69 +87,33 @@ def building_class(BIM, hazard): bldg_class = 'WMUH' elif BIM['BuildingType'] == "Steel": - if (BIM['DesignLevel'] == 'E') and ( - BIM['OccupancyClass'] - in ['RES3A', 'RES3B', 'RES3C', 'RES3D', 'RES3E', 'RES3F'] - ): + if ((BIM['DesignLevel'] == 'E') and + (BIM['OccupancyClass'] in ['RES3A', 'RES3B', 'RES3C', 'RES3D', + 'RES3E', 'RES3F'])): # Steel Engineered Residential Building (SERBL, SERBM, SERBH) bldg_class = 'SERB' - elif (BIM['DesignLevel'] == 'E') and ( - BIM['OccupancyClass'] - in [ - 'COM1', - 'COM2', - 'COM3', - 'COM4', - 'COM5', - 'COM6', - 'COM7', - 'COM8', - 'COM9', - 'COM10', - ] - ): + elif ((BIM['DesignLevel'] == 'E') and + (BIM['OccupancyClass'] in ['COM1', 'COM2', 'COM3', 'COM4', 'COM5', + 'COM6', 'COM7', 'COM8', 'COM9','COM10'])): # Steel Engineered Commercial Building (SECBL, SECBM, SECBH) bldg_class = 'SECB' - elif (BIM['DesignLevel'] == 'PE') and ( - BIM['OccupancyClass'] - not in ['RES3A', 'RES3B', 'RES3C', 'RES3D', 'RES3E', 'RES3F'] - ): + elif ((BIM['DesignLevel'] == 'PE') and + (BIM['OccupancyClass'] not in ['RES3A', 'RES3B', 'RES3C', 'RES3D', + 'RES3E', 'RES3F'])): # Steel Pre-Engineered Metal Building (SPMBS, SPMBM, SPMBL) bldg_class = 'SPMB' else: bldg_class = 'SECB' elif BIM['BuildingType'] == "Concrete": - if (BIM['DesignLevel'] == 'E') and ( - BIM['OccupancyClass'] - in [ - 'RES3A', - 'RES3B', - 'RES3C', - 'RES3D', - 'RES3E', - 'RES3F', - 'RES5', - 'RES6', - ] - ): + if ((BIM['DesignLevel'] == 'E') and + (BIM['OccupancyClass'] in ['RES3A', 'RES3B', 'RES3C', 'RES3D', + 'RES3E', 'RES3F', 'RES5', 'RES6'])): # Concrete Engineered Residential Building (CERBL, CERBM, CERBH) bldg_class = 'CERB' - elif (BIM['DesignLevel'] == 'E') and ( - BIM['OccupancyClass'] - in [ - 'COM1', - 'COM2', - 'COM3', - 'COM4', - 'COM5', - 'COM6', - 'COM7', - 'COM8', - 'COM9', - 'COM10', - ] - ): + elif ((BIM['DesignLevel'] == 'E') and + (BIM['OccupancyClass'] in ['COM1', 'COM2', 'COM3', 'COM4', 'COM5', + 'COM6', 'COM7', 'COM8', 'COM9','COM10'])): # Concrete Engineered Commercial Building (CECBL, CECBM, CECBH) bldg_class = 'CECB' else: @@ -164,77 +124,35 @@ def building_class(BIM, hazard): # OccupancyClass = RES1 # Masonry Single-Family Homes (MSF1 or MSF2) bldg_class = 'MSF' - elif ( - BIM['OccupancyClass'] - in ['RES3A', 'RES3B', 'RES3C', 'RES3D', 'RES3E', 'RES3F'] - ) and (BIM['DesignLevel'] == 'E'): + elif ((BIM['OccupancyClass'] in ['RES3A', 'RES3B', 'RES3C', 'RES3D', + 'RES3E', 'RES3F']) and (BIM['DesignLevel'] == 'E')): # Masonry Engineered Residential Building (MERBL, MERBM, MERBH) bldg_class = 'MERB' - elif ( - BIM['OccupancyClass'] - in [ - 'COM1', - 'COM2', - 'COM3', - 'COM4', - 'COM5', - 'COM6', - 'COM7', - 'COM8', - 'COM9', - 'COM10', - ] - ) and (BIM['DesignLevel'] == 'E'): + elif ((BIM['OccupancyClass'] in ['COM1', 'COM2', 'COM3', 'COM4', + 'COM5', 'COM6', 'COM7', 'COM8', 'COM9', + 'COM10']) and (BIM['DesignLevel'] == 'E')): # Masonry Engineered Commercial Building (MECBL, MECBM, MECBH) bldg_class = 'MECB' - elif BIM['OccupancyClass'] in [ - 'IND1', - 'IND2', - 'IND3', - 'IND4', - 'IND5', - 'IND6', - ]: + elif BIM['OccupancyClass'] in ['IND1', 'IND2', 'IND3', 'IND4', 'IND5', 'IND6']: # Masonry Low-Rise Masonry Warehouse/Factory (MLRI) bldg_class = 'MLRI' - elif BIM['OccupancyClass'] in [ - 'RES3A', - 'RES3B', - 'RES3C', - 'RES3D', - 'RES3E', - 'RES3F', - 'RES5', - 'RES6', - 'COM8', - ]: + elif BIM['OccupancyClass'] in ['RES3A', 'RES3B', 'RES3C', 'RES3D', + 'RES3E', 'RES3F', 'RES5', 'RES6', 'COM8']: # OccupancyClass = RES3X or COM8 # Masonry Multi-Unit Hotel/Motel (MMUH1, MMUH2, or MMUH3) bldg_class = 'MMUH' - elif (BIM['NumberOfStories'] == 1) and ( - BIM['OccupancyClass'] in ['COM1', 'COM2'] - ): + elif ((BIM['NumberOfStories'] == 1) and + (BIM['OccupancyClass'] in ['COM1', 'COM2'])): # Low-Rise Masonry Strip Mall (MLRM1 or MLRM2) bldg_class = 'MLRM' - # elif ( - # BIM['OccupancyClass'] - # in [ - # 'RES3A', - # 'RES3B', - # 'RES3C', - # 'RES3D', - # 'RES3E', - # 'RES3F', - # 'RES5', - # 'RES6', - # 'COM8', - # ] - # ) and (BIM['DesignLevel'] in ['NE', 'ME']): - # # Masonry Multi-Unit Hotel/Motel Non-Engineered - # # (MMUH1NE, MMUH2NE, or MMUH3NE) - # bldg_class = 'MMUHNE' else: - bldg_class = 'MECB' # for others not covered by the above + bldg_class = 'MECB' # for others not covered by the above + #elif ((BIM['OccupancyClass'] in ['RES3A', 'RES3B', 'RES3C', 'RES3D', + # 'RES3E', 'RES3F', 'RES5', 'RES6', + # 'COM8']) and (BIM['DesignLevel'] in ['NE', 'ME'])): + # # Masonry Multi-Unit Hotel/Motel Non-Engineered + # # (MMUH1NE, MMUH2NE, or MMUH3NE) + # bldg_class = 'MMUHNE' elif BIM['BuildingType'] == "Manufactured": bldg_class = 'MH' diff --git a/pelicun/tests/dl_calculation/rulesets/BuildingClassRulesets.py b/pelicun/tests/dl_calculation/rulesets/BuildingClassRulesets.py index 40312a8b1..b646946f0 100644 --- a/pelicun/tests/dl_calculation/rulesets/BuildingClassRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/BuildingClassRulesets.py @@ -43,6 +43,10 @@ # Meredith Lockhead # Tracy Kijewski-Correa +import random +import numpy as np +import datetime + def building_class(BIM, hazard): """ @@ -68,22 +72,15 @@ def building_class(BIM, hazard): if hazard == 'wind': if BIM['BuildingType'] == 'Wood': - if (BIM['OccupancyClass'] == 'RES1') or ( - (BIM['RoofShape'] != 'flt') and (BIM['OccupancyClass'] == '') - ): + if ((BIM['OccupancyClass'] == 'RES1') or + ((BIM['RoofShape'] != 'flt') and (BIM['OccupancyClass'] == ''))): # BuildingType = 3001 # OccupancyClass = RES1 # Wood Single-Family Homes (WSF1 or WSF2) # OR roof type = flat (HAZUS can only map flat to WSF1) # OR default (by '') - if ( - BIM['RoofShape'] == 'flt' - ): # checking if there is a misclassication - BIM['RoofShape'] = ( - # ensure the WSF has gab (by default, note gab - # is more vulneable than hip) - 'gab' - ) + if BIM['RoofShape'] == 'flt': # checking if there is a misclassication + BIM['RoofShape'] = 'gab' # ensure the WSF has gab (by default, note gab is more vulneable than hip) bldg_class = 'WSF' else: # BuildingType = 3001 @@ -91,72 +88,36 @@ def building_class(BIM, hazard): # Wood Multi-Unit Hotel (WMUH1, WMUH2, or WMUH3) bldg_class = 'WMUH' elif BIM['BuildingType'] == 'Steel': - if (BIM['DesignLevel'] == 'E') and ( - BIM['OccupancyClass'] - in ['RES3A', 'RES3B', 'RES3C', 'RES3D', 'RES3E', 'RES3F'] - ): + if ((BIM['DesignLevel'] == 'E') and + (BIM['OccupancyClass'] in ['RES3A', 'RES3B', 'RES3C', 'RES3D', + 'RES3E', 'RES3F'])): # BuildingType = 3002 # Steel Engineered Residential Building (SERBL, SERBM, SERBH) bldg_class = 'SERB' - elif (BIM['DesignLevel'] == 'E') and ( - BIM['OccupancyClass'] - in [ - 'COM1', - 'COM2', - 'COM3', - 'COM4', - 'COM5', - 'COM6', - 'COM7', - 'COM8', - 'COM9', - 'COM10', - ] - ): + elif ((BIM['DesignLevel'] == 'E') and + (BIM['OccupancyClass'] in ['COM1', 'COM2', 'COM3', 'COM4', 'COM5', + 'COM6', 'COM7', 'COM8', 'COM9','COM10'])): # BuildingType = 3002 # Steel Engineered Commercial Building (SECBL, SECBM, SECBH) bldg_class = 'SECB' - elif (BIM['DesignLevel'] == 'PE') and ( - BIM['OccupancyClass'] - not in ['RES3A', 'RES3B', 'RES3C', 'RES3D', 'RES3E', 'RES3F'] - ): + elif ((BIM['DesignLevel'] == 'PE') and + (BIM['OccupancyClass'] not in ['RES3A', 'RES3B', 'RES3C', 'RES3D', + 'RES3E', 'RES3F'])): # BuildingType = 3002 # Steel Pre-Engineered Metal Building (SPMBS, SPMBM, SPMBL) bldg_class = 'SPMB' else: bldg_class = 'SECB' elif BIM['BuildingType'] == 'Concrete': - if (BIM['DesignLevel'] == 'E') and ( - BIM['OccupancyClass'] - in [ - 'RES3A', - 'RES3B', - 'RES3C', - 'RES3D', - 'RES3E', - 'RES3F', - 'RES5', - 'RES6', - ] - ): + if ((BIM['DesignLevel'] == 'E') and + (BIM['OccupancyClass'] in ['RES3A', 'RES3B', 'RES3C', 'RES3D', + 'RES3E', 'RES3F', 'RES5', 'RES6'])): # BuildingType = 3003 # Concrete Engineered Residential Building (CERBL, CERBM, CERBH) bldg_class = 'CERB' - elif (BIM['DesignLevel'] == 'E') and ( - BIM['OccupancyClass'] - in [ - 'COM1', - 'COM2', - 'COM3', - 'COM4', - 'COM5', - 'COM6', - 'COM7', - 'COM8', - 'COM9', - 'COM10', - ] - ): + elif ((BIM['DesignLevel'] == 'E') and + (BIM['OccupancyClass'] in ['COM1', 'COM2', 'COM3', 'COM4', 'COM5', + 'COM6', 'COM7', 'COM8', 'COM9','COM10'])): # BuildingType = 3003 # Concrete Engineered Commercial Building (CECBL, CECBM, CECBH) bldg_class = 'CECB' @@ -168,83 +129,41 @@ def building_class(BIM, hazard): # OccupancyClass = RES1 # Masonry Single-Family Homes (MSF1 or MSF2) bldg_class = 'MSF' - elif ( - BIM['OccupancyClass'] - in ['RES3A', 'RES3B', 'RES3C', 'RES3D', 'RES3E', 'RES3F'] - ) and (BIM['DesignLevel'] == 'E'): + elif ((BIM['OccupancyClass'] in ['RES3A', 'RES3B', 'RES3C', 'RES3D', + 'RES3E', 'RES3F']) and (BIM['DesignLevel'] == 'E')): # BuildingType = 3004 # Masonry Engineered Residential Building (MERBL, MERBM, MERBH) bldg_class = 'MERB' - elif ( - BIM['OccupancyClass'] - in [ - 'COM1', - 'COM2', - 'COM3', - 'COM4', - 'COM5', - 'COM6', - 'COM7', - 'COM8', - 'COM9', - 'COM10', - ] - ) and (BIM['DesignLevel'] == 'E'): + elif ((BIM['OccupancyClass'] in ['COM1', 'COM2', 'COM3', 'COM4', + 'COM5', 'COM6', 'COM7', 'COM8', 'COM9', + 'COM10']) and (BIM['DesignLevel'] == 'E')): # BuildingType = 3004 # Masonry Engineered Commercial Building (MECBL, MECBM, MECBH) bldg_class = 'MECB' - elif BIM['OccupancyClass'] in [ - 'IND1', - 'IND2', - 'IND3', - 'IND4', - 'IND5', - 'IND6', - ]: + elif BIM['OccupancyClass'] in ['IND1', 'IND2', 'IND3', 'IND4', 'IND5', 'IND6']: # BuildingType = 3004 # Masonry Low-Rise Masonry Warehouse/Factory (MLRI) bldg_class = 'MLRI' - elif BIM['OccupancyClass'] in [ - 'RES3A', - 'RES3B', - 'RES3C', - 'RES3D', - 'RES3E', - 'RES3F', - 'RES5', - 'RES6', - 'COM8', - ]: + elif BIM['OccupancyClass'] in ['RES3A', 'RES3B', 'RES3C', 'RES3D', + 'RES3E', 'RES3F', 'RES5', 'RES6', 'COM8']: # BuildingType = 3004 # OccupancyClass = RES3X or COM8 # Masonry Multi-Unit Hotel/Motel (MMUH1, MMUH2, or MMUH3) bldg_class = 'MMUH' - elif (BIM['NumberOfStories'] == 1) and ( - BIM['OccupancyClass'] in ['COM1', 'COM2'] - ): + elif ((BIM['NumberOfStories'] == 1) and + (BIM['OccupancyClass'] in ['COM1', 'COM2'])): # BuildingType = 3004 # Low-Rise Masonry Strip Mall (MLRM1 or MLRM2) bldg_class = 'MLRM' - # elif ( - # BIM['OccupancyClass'] - # in [ - # 'RES3A', - # 'RES3B', - # 'RES3C', - # 'RES3D', - # 'RES3E', - # 'RES3F', - # 'RES5', - # 'RES6', - # 'COM8', - # ] - # ) and (BIM['DesignLevel'] in ['NE', 'ME']): - # # BuildingType = 3004 - # # Masonry Multi-Unit Hotel/Motel Non-Engineered - # # (MMUH1NE, MMUH2NE, or MMUH3NE) - # return 'MMUHNE' else: - bldg_class = 'MECB' # for others not covered by the above + bldg_class = 'MECB' # for others not covered by the above + #elif ((BIM['OccupancyClass'] in ['RES3A', 'RES3B', 'RES3C', 'RES3D', + # 'RES3E', 'RES3F', 'RES5', 'RES6', + # 'COM8']) and (BIM['DesignLevel'] in ['NE', 'ME'])): + # # BuildingType = 3004 + # # Masonry Multi-Unit Hotel/Motel Non-Engineered + # # (MMUH1NE, MMUH2NE, or MMUH3NE) + # return 'MMUHNE' elif BIM['BuildingType'] == 'Manufactured': bldg_class = 'MH' @@ -252,4 +171,4 @@ def building_class(BIM, hazard): bldg_class = 'WMUH' # if nan building type is provided, return the dominant class - return bldg_class + return bldg_class \ No newline at end of file diff --git a/pelicun/tests/dl_calculation/rulesets/FloodAssmRulesets.py b/pelicun/tests/dl_calculation/rulesets/FloodAssmRulesets.py index c9fecb11d..658d2e4a3 100644 --- a/pelicun/tests/dl_calculation/rulesets/FloodAssmRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/FloodAssmRulesets.py @@ -44,6 +44,10 @@ # Meredith Lockhead # Tracy Kijewski-Correa +import random +import numpy as np +import datetime +import math def Assm_config(BIM): """ @@ -60,84 +64,42 @@ def Assm_config(BIM): A string that identifies a specific configration within this buidling class. """ - year = BIM['YearBuilt'] # just for the sake of brevity + year = BIM['YearBuilt'] # just for the sake of brevity # Flood Type if BIM['FloodZone'] in ['AO']: - flood_type = 'raz' # Riverline/A-Zone + flood_type = 'raz' # Riverline/A-Zone elif BIM['FloodZone'] in ['AE', 'AH', 'A']: - flood_type = 'caz' # Costal/A-Zone + flood_type = 'caz' # Costal/A-Zone elif BIM['FloodZone'] in ['VE']: - flood_type = 'cvz' # Costal/V-Zone + flood_type = 'cvz' # Costal/V-Zone else: - flood_type = 'caz' # Default + flood_type = 'caz' # Default # PostFIRM - PostFIRM = False # Default - city_list = [ - 'Absecon', - 'Atlantic', - 'Brigantine', - 'Buena', - 'Buena Vista', - 'Corbin City', - 'Egg Harbor City', - 'Egg Harbor', - 'Estell Manor', - 'Folsom', - 'Galloway', - 'Hamilton', - 'Hammonton', - 'Linwood', - 'Longport', - 'Margate City', - 'Mullica', - 'Northfield', - 'Pleasantville', - 'Port Republic', - 'Somers Point', - 'Ventnor City', - 'Weymouth', - ] - year_list = [ - 1976, - 1971, - 1971, - 1983, - 1979, - 1981, - 1982, - 1983, - 1978, - 1982, - 1983, - 1977, - 1982, - 1983, - 1974, - 1974, - 1982, - 1979, - 1983, - 1983, - 1982, - 1971, - 1979, - ] - for i in range(0, 22): - PostFIRM = ( - (BIM['City'] == city_list[i]) and (year > year_list[i]) - ) or PostFIRM + PostFIRM = False # Default + city_list = ['Absecon', 'Atlantic', 'Brigantine', 'Buena', 'Buena Vista', + 'Corbin City', 'Egg Harbor City', 'Egg Harbor', 'Estell Manor', + 'Folsom', 'Galloway', 'Hamilton', 'Hammonton', 'Linwood', + 'Longport', 'Margate City', 'Mullica', 'Northfield', + 'Pleasantville', 'Port Republic', 'Somers Point', + 'Ventnor City', 'Weymouth'] + year_list = [1976, 1971, 1971, 1983, 1979, 1981, 1982, 1983, 1978, 1982, + 1983, 1977, 1982, 1983, 1974, 1974, 1982, 1979, 1983, 1983, + 1982, 1971, 1979] + for i in range(0,22): + PostFIRM = (((BIM['City'] == city_list[i]) and (year > year_list[i])) or \ + PostFIRM) # fl_assm - fl_assm = ( - f"{'fl_surge_assm'}_" - f"{BIM['OccupancyClass']}_" - f"{int(PostFIRM)}_" - f"{flood_type}" - ) + fl_assm = f"{'fl_surge_assm'}_" \ + f"{BIM['OccupancyClass']}_" \ + f"{int(PostFIRM)}_" \ + f"{flood_type}" # hu_assm - hu_assm = f"{'hu_surge_assm'}_" f"{BIM['OccupancyClass']}_" f"{int(PostFIRM)}" + hu_assm = f"{'hu_surge_assm'}_" \ + f"{BIM['OccupancyClass']}_" \ + f"{int(PostFIRM)}" - return hu_assm, fl_assm + return hu_assm, fl_assm \ No newline at end of file diff --git a/pelicun/tests/dl_calculation/rulesets/FloodClassRulesets.py b/pelicun/tests/dl_calculation/rulesets/FloodClassRulesets.py index 7bbcb94b0..702c829ec 100644 --- a/pelicun/tests/dl_calculation/rulesets/FloodClassRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/FloodClassRulesets.py @@ -45,7 +45,6 @@ import numpy as np - def FL_config(BIM): """ Rules to identify the flood vunerability category @@ -61,186 +60,140 @@ def FL_config(BIM): A string that identifies a specific configration within this buidling class. """ - year = BIM['YearBuilt'] # just for the sake of brevity + year = BIM['YearBuilt'] # just for the sake of brevity # Flood Type if BIM['FloodZone'] == 'AO': - flood_type = 'raz' # Riverline/A-Zone + flood_type = 'raz' # Riverline/A-Zone elif BIM['FloodZone'] in ['A', 'AE']: - flood_type = 'cvz' # Costal-Zone + flood_type = 'cvz' # Costal-Zone elif BIM['FloodZone'].startswith('V'): - flood_type = 'cvz' # Costal-Zone + flood_type = 'cvz' # Costal-Zone else: - flood_type = 'cvz' # Default + flood_type = 'cvz' # Default - # flake8 - unused variable: `FFE` - # # First Floor Elevation (FFE) - # if flood_type in ['raz', 'caz']: - # FFE = BIM['FirstFloorElevation'] - # else: - # FFE = BIM['FirstFloorElevation'] - 1.0 + # First Floor Elevation (FFE) + if flood_type in ['raz', 'caz']: + FFE = BIM['FirstFloorElevation'] + else: + FFE = BIM['FirstFloorElevation'] - 1.0 # PostFIRM - PostFIRM = False # Default - city_list = [ - 'Absecon', - 'Atlantic', - 'Brigantine', - 'Buena', - 'Buena Vista', - 'Corbin City', - 'Egg Harbor City', - 'Egg Harbor', - 'Estell Manor', - 'Folsom', - 'Galloway', - 'Hamilton', - 'Hammonton', - 'Linwood', - 'Longport', - 'Margate City', - 'Mullica', - 'Northfield', - 'Pleasantville', - 'Port Republic', - 'Somers Point', - 'Ventnor City', - 'Weymouth', - ] - year_list = [ - 1976, - 1971, - 1971, - 1983, - 1979, - 1981, - 1982, - 1983, - 1978, - 1982, - 1983, - 1977, - 1982, - 1983, - 1974, - 1974, - 1982, - 1979, - 1983, - 1983, - 1982, - 1971, - 1979, - ] - for i in range(0, 22): - PostFIRM = ( - (BIM['City'] == city_list[i]) and (year > year_list[i]) - ) or PostFIRM + PostFIRM = False # Default + city_list = ['Absecon', 'Atlantic', 'Brigantine', 'Buena', 'Buena Vista', + 'Corbin City', 'Egg Harbor City', 'Egg Harbor', 'Estell Manor', + 'Folsom', 'Galloway', 'Hamilton', 'Hammonton', 'Linwood', + 'Longport', 'Margate City', 'Mullica', 'Northfield', + 'Pleasantville', 'Port Republic', 'Somers Point', + 'Ventnor City', 'Weymouth'] + year_list = [1976, 1971, 1971, 1983, 1979, 1981, 1982, 1983, 1978, 1982, + 1983, 1977, 1982, 1983, 1974, 1974, 1982, 1979, 1983, 1983, + 1982, 1971, 1979] + for i in range(0,22): + PostFIRM = (((BIM['City'] == city_list[i]) and (year > year_list[i])) or \ + PostFIRM) # Basement Type if BIM['SplitLevel'] and (BIM['FoundationType'] == 3504): - bmt_type = 'spt' # Split-Level Basement + bmt_type = 'spt' # Split-Level Basement elif BIM['FoundationType'] in [3501, 3502, 3503, 3505, 3506, 3507]: - bmt_type = 'bn' # No Basement + bmt_type = 'bn' # No Basement elif (not BIM['SplitLevel']) and (BIM['FoundationType'] == 3504): - bmt_type = 'bw' # Basement + bmt_type = 'bw' # Basement else: - bmt_type = 'bw' # Default + bmt_type = 'bw' # Default + + # Duration + dur = 'short' - # flake8 - unused variable: `dur`. - # # Duration - # dur = 'short' + # Occupancy Type + if BIM['OccupancyClass'] == 'RES1': + if BIM['NumberOfStories'] == 1: + if flood_type == 'raz': + OT = 'SF1XA' + elif flood_type == 'cvz': + OT = 'SF1XV' + else: + if bmt_type == 'nav': + if flood_type == 'raz': + OT = 'SF2XA' + elif flood_type == 'cvz': + OT = 'SF2XV' + elif bmt_type == 'bmt': + if flood_type == 'raz': + OT = 'SF2BA' + elif flood_type == 'cvz': + OT = 'SF2BV' + elif bmt_type == 'spt': + if flood_type == 'raz': + OT = 'SF2SA' + elif flood_type == 'cvz': + OT = 'SF2SV' + elif 'RES3' in BIM['OccupancyClass']: + OT = 'APT' + else: + ap_OT = { + 'RES2': 'MH', + 'RES4': 'HOT', + 'RES5': 'NURSE', + 'RES6': 'NURSE', + 'COM1': 'RETAL', + 'COM2': 'WHOLE', + 'COM3': 'SERVICE', + 'COM4': 'OFFICE', + 'COM5': 'BANK', + 'COM6': 'HOSP', + 'COM7': 'MED', + 'COM8': 'REC', + 'COM9': 'THEAT', + 'COM10': 'GARAGE', + 'IND1': 'INDH', + 'IND2': 'INDL', + 'IND3': 'CHEM', + 'IND4': 'PROC', + 'IND5': 'CHEM', + 'IND6': 'CONST', + 'AGR1': 'AGRI', + 'REL1': 'RELIG', + 'GOV1': 'CITY', + 'GOV2': 'EMERG', + 'EDU1': 'SCHOOL', + 'EDU2': 'SCHOOL' + } + ap_OT[BIM['OccupancyClass']] - # flake8: unused variable: `OT` - # # Occupancy Type - # if BIM['OccupancyClass'] == 'RES1': - # if BIM['NumberOfStories'] == 1: - # if flood_type == 'raz': - # OT = 'SF1XA' - # elif flood_type == 'cvz': - # OT = 'SF1XV' - # else: - # if bmt_type == 'nav': - # if flood_type == 'raz': - # OT = 'SF2XA' - # elif flood_type == 'cvz': - # OT = 'SF2XV' - # elif bmt_type == 'bmt': - # if flood_type == 'raz': - # OT = 'SF2BA' - # elif flood_type == 'cvz': - # OT = 'SF2BV' - # elif bmt_type == 'spt': - # if flood_type == 'raz': - # OT = 'SF2SA' - # elif flood_type == 'cvz': - # OT = 'SF2SV' - # elif 'RES3' in BIM['OccupancyClass']: - # OT = 'APT' - # else: - # ap_OT = { - # 'RES2': 'MH', - # 'RES4': 'HOT', - # 'RES5': 'NURSE', - # 'RES6': 'NURSE', - # 'COM1': 'RETAL', - # 'COM2': 'WHOLE', - # 'COM3': 'SERVICE', - # 'COM4': 'OFFICE', - # 'COM5': 'BANK', - # 'COM6': 'HOSP', - # 'COM7': 'MED', - # 'COM8': 'REC', - # 'COM9': 'THEAT', - # 'COM10': 'GARAGE', - # 'IND1': 'INDH', - # 'IND2': 'INDL', - # 'IND3': 'CHEM', - # 'IND4': 'PROC', - # 'IND5': 'CHEM', - # 'IND6': 'CONST', - # 'AGR1': 'AGRI', - # 'REL1': 'RELIG', - # 'GOV1': 'CITY', - # 'GOV2': 'EMERG', - # 'EDU1': 'SCHOOL', - # 'EDU2': 'SCHOOL', - # } - # ap_OT[BIM['OccupancyClass']] if not (BIM['OccupancyClass'] in ['RES1', 'RES2']): if 'RES3' in BIM['OccupancyClass']: - fl_config = f"{'fl'}_" f"{'RES3'}" + fl_config = f"{'fl'}_" \ + f"{'RES3'}" else: - fl_config = f"{'fl'}_" f"{BIM['OccupancyClass']}" + fl_config = f"{'fl'}_" \ + f"{BIM['OccupancyClass']}" elif BIM['OccupancyClass'] == 'RES2': - fl_config = f"{'fl'}_" f"{BIM['OccupancyClass']}_" f"{flood_type}" + fl_config = f"{'fl'}_" \ + f"{BIM['OccupancyClass']}_" \ + f"{flood_type}" else: if bmt_type == 'spt': - fl_config = ( - f"{'fl'}_" - f"{BIM['OccupancyClass']}_" - f"{'sl'}_" - f"{'bw'}_" - f"{flood_type}" - ) + fl_config = f"{'fl'}_" \ + f"{BIM['OccupancyClass']}_" \ + f"{'sl'}_" \ + f"{'bw'}_" \ + f"{flood_type}" else: - st = 's' + str(np.min([BIM['NumberOfStories'], 3])) - fl_config = ( - f"{'fl'}_" - f"{BIM['OccupancyClass']}_" - f"{st}_" - f"{bmt_type}_" - f"{flood_type}" - ) + st = 's'+str(np.min([BIM['NumberOfStories'],3])) + fl_config = f"{'fl'}_" \ + f"{BIM['OccupancyClass']}_" \ + f"{st}_" \ + f"{bmt_type}_" \ + f"{flood_type}" # extend the BIM dictionary - BIM.update( - dict( - FloodType=flood_type, - BasementType=bmt_type, - PostFIRM=PostFIRM, - ) - ) + BIM.update(dict( + FloodType = flood_type, + BasementType=bmt_type, + PostFIRM=PostFIRM, + )) - return fl_config + return fl_config \ No newline at end of file diff --git a/pelicun/tests/dl_calculation/rulesets/FloodRulesets.py b/pelicun/tests/dl_calculation/rulesets/FloodRulesets.py index d195bef7a..882d8d933 100644 --- a/pelicun/tests/dl_calculation/rulesets/FloodRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/FloodRulesets.py @@ -45,7 +45,6 @@ import numpy as np - def FL_config(BIM): """ Rules to identify the flood vunerability category @@ -61,186 +60,141 @@ def FL_config(BIM): A string that identifies a specific configration within this buidling class. """ - year = BIM['YearBuilt'] # just for the sake of brevity + year = BIM['YearBuilt'] # just for the sake of brevity # Flood Type if BIM['FloodZone'] == 'AO': - flood_type = 'raz' # Riverline/A-Zone + flood_type = 'raz' # Riverline/A-Zone elif BIM['FloodZone'] in ['A', 'AE']: - flood_type = 'cvz' # Costal-Zone + flood_type = 'cvz' # Costal-Zone elif BIM['FloodZone'].startswith('V'): - flood_type = 'cvz' # Costal-Zone + flood_type = 'cvz' # Costal-Zone else: - flood_type = 'cvz' # Default + flood_type = 'cvz' # Default - # flake8 - unused variable: `FFE`. - # # First Floor Elevation (FFE) - # if flood_type in ['raz', 'caz']: - # FFE = BIM['FirstFloorElevation'] - # else: - # FFE = BIM['FirstFloorElevation'] - 1.0 + # First Floor Elevation (FFE) + if flood_type in ['raz', 'caz']: + FFE = BIM['FirstFloorElevation'] + else: + FFE = BIM['FirstFloorElevation'] - 1.0 # PostFIRM - PostFIRM = False # Default - city_list = [ - 'Absecon', - 'Atlantic', - 'Brigantine', - 'Buena', - 'Buena Vista', - 'Corbin City', - 'Egg Harbor City', - 'Egg Harbor', - 'Estell Manor', - 'Folsom', - 'Galloway', - 'Hamilton', - 'Hammonton', - 'Linwood', - 'Longport', - 'Margate City', - 'Mullica', - 'Northfield', - 'Pleasantville', - 'Port Republic', - 'Somers Point', - 'Ventnor City', - 'Weymouth', - ] - year_list = [ - 1976, - 1971, - 1971, - 1983, - 1979, - 1981, - 1982, - 1983, - 1978, - 1982, - 1983, - 1977, - 1982, - 1983, - 1974, - 1974, - 1982, - 1979, - 1983, - 1983, - 1982, - 1971, - 1979, - ] - for i in range(0, 22): - PostFIRM = ( - (BIM['City'] == city_list[i]) and (year > year_list[i]) - ) or PostFIRM + PostFIRM = False # Default + city_list = ['Absecon', 'Atlantic', 'Brigantine', 'Buena', 'Buena Vista', + 'Corbin City', 'Egg Harbor City', 'Egg Harbor', 'Estell Manor', + 'Folsom', 'Galloway', 'Hamilton', 'Hammonton', 'Linwood', + 'Longport', 'Margate City', 'Mullica', 'Northfield', + 'Pleasantville', 'Port Republic', 'Somers Point', + 'Ventnor City', 'Weymouth'] + year_list = [1976, 1971, 1971, 1983, 1979, 1981, 1982, 1983, 1978, 1982, + 1983, 1977, 1982, 1983, 1974, 1974, 1982, 1979, 1983, 1983, + 1982, 1971, 1979] + for i in range(0,22): + PostFIRM = (((BIM['City'] == city_list[i]) and (year > year_list[i])) or \ + PostFIRM) # Basement Type if BIM['SplitLevel'] and (BIM['FoundationType'] == 3504): - bmt_type = 'spt' # Split-Level Basement + bmt_type = 'spt' # Split-Level Basement elif BIM['FoundationType'] in [3501, 3502, 3503, 3505, 3506, 3507]: - bmt_type = 'bn' # No Basement + bmt_type = 'bn' # No Basement elif (not BIM['SplitLevel']) and (BIM['FoundationType'] == 3504): - bmt_type = 'bw' # Basement + bmt_type = 'bw' # Basement else: - bmt_type = 'bw' # Default + bmt_type = 'bw' # Default + + # Duration + dur = 'short' - # flake8 - unused variable: `dur`. - # # Duration - # dur = 'short' + # Occupancy Type + if BIM['OccupancyClass'] == 'RES1': + if BIM['NumberOfStories'] == 1: + if flood_type == 'raz': + OT = 'SF1XA' + elif flood_type == 'cvz': + OT = 'SF1XV' + else: + if bmt_type == 'nav': + if flood_type == 'raz': + OT = 'SF2XA' + elif flood_type == 'cvz': + OT = 'SF2XV' + elif bmt_type == 'bmt': + if flood_type == 'raz': + OT = 'SF2BA' + elif flood_type == 'cvz': + OT = 'SF2BV' + elif bmt_type == 'spt': + if flood_type == 'raz': + OT = 'SF2SA' + elif flood_type == 'cvz': + OT = 'SF2SV' + elif 'RES3' in BIM['OccupancyClass']: + OT = 'APT' + else: + ap_OT = { + 'RES2': 'MH', + 'RES4': 'HOT', + 'RES5': 'NURSE', + 'RES6': 'NURSE', + 'COM1': 'RETAL', + 'COM2': 'WHOLE', + 'COM3': 'SERVICE', + 'COM4': 'OFFICE', + 'COM5': 'BANK', + 'COM6': 'HOSP', + 'COM7': 'MED', + 'COM8': 'REC', + 'COM9': 'THEAT', + 'COM10': 'GARAGE', + 'IND1': 'INDH', + 'IND2': 'INDL', + 'IND3': 'CHEM', + 'IND4': 'PROC', + 'IND5': 'CHEM', + 'IND6': 'CONST', + 'AGR1': 'AGRI', + 'REL1': 'RELIG', + 'GOV1': 'CITY', + 'GOV2': 'EMERG', + 'EDU1': 'SCHOOL', + 'EDU2': 'SCHOOL' + } + ap_OT[BIM['OccupancyClass']] - # flake8 - unused variable: `OT`. - # # Occupancy Type - # if BIM['OccupancyClass'] == 'RES1': - # if BIM['NumberOfStories'] == 1: - # if flood_type == 'raz': - # OT = 'SF1XA' - # elif flood_type == 'cvz': - # OT = 'SF1XV' - # else: - # if bmt_type == 'nav': - # if flood_type == 'raz': - # OT = 'SF2XA' - # elif flood_type == 'cvz': - # OT = 'SF2XV' - # elif bmt_type == 'bmt': - # if flood_type == 'raz': - # OT = 'SF2BA' - # elif flood_type == 'cvz': - # OT = 'SF2BV' - # elif bmt_type == 'spt': - # if flood_type == 'raz': - # OT = 'SF2SA' - # elif flood_type == 'cvz': - # OT = 'SF2SV' - # elif 'RES3' in BIM['OccupancyClass']: - # OT = 'APT' - # else: - # ap_OT = { - # 'RES2': 'MH', - # 'RES4': 'HOT', - # 'RES5': 'NURSE', - # 'RES6': 'NURSE', - # 'COM1': 'RETAL', - # 'COM2': 'WHOLE', - # 'COM3': 'SERVICE', - # 'COM4': 'OFFICE', - # 'COM5': 'BANK', - # 'COM6': 'HOSP', - # 'COM7': 'MED', - # 'COM8': 'REC', - # 'COM9': 'THEAT', - # 'COM10': 'GARAGE', - # 'IND1': 'INDH', - # 'IND2': 'INDL', - # 'IND3': 'CHEM', - # 'IND4': 'PROC', - # 'IND5': 'CHEM', - # 'IND6': 'CONST', - # 'AGR1': 'AGRI', - # 'REL1': 'RELIG', - # 'GOV1': 'CITY', - # 'GOV2': 'EMERG', - # 'EDU1': 'SCHOOL', - # 'EDU2': 'SCHOOL', - # } - # ap_OT[BIM['OccupancyClass']] if not (BIM['OccupancyClass'] in ['RES1', 'RES2']): if 'RES3' in BIM['OccupancyClass']: - fl_config = f"{'fl'}_" f"{'RES3'}" + fl_config = f"{'fl'}_" \ + f"{'RES3'}" else: - fl_config = f"{'fl'}_" f"{BIM['OccupancyClass']}" + fl_config = f"{'fl'}_" \ + f"{BIM['OccupancyClass']}" elif BIM['OccupancyClass'] == 'RES2': - fl_config = f"{'fl'}_" f"{BIM['OccupancyClass']}_" f"{flood_type}" + fl_config = f"{'fl'}_" \ + f"{BIM['OccupancyClass']}_" \ + f"{flood_type}" else: if bmt_type == 'spt': - fl_config = ( - f"{'fl'}_" - f"{BIM['OccupancyClass']}_" - f"{'sl'}_" - f"{'bw'}_" - f"{flood_type}" - ) + fl_config = f"{'fl'}_" \ + f"{BIM['OccupancyClass']}_" \ + f"{'sl'}_" \ + f"{'bw'}_" \ + f"{flood_type}" else: - st = 's' + str(np.min([BIM['NumberOfStories'], 3])) - fl_config = ( - f"{'fl'}_" - f"{BIM['OccupancyClass']}_" - f"{st}_" - f"{bmt_type}_" - f"{flood_type}" - ) + st = 's'+str(np.min([BIM['NumberOfStories'],3])) + fl_config = f"{'fl'}_" \ + f"{BIM['OccupancyClass']}_" \ + f"{st}_" \ + f"{bmt_type}_" \ + f"{flood_type}" # extend the BIM dictionary - BIM.update( - dict( - FloodType=flood_type, - BasementType=bmt_type, - PostFIRM=PostFIRM, - ) - ) + BIM.update(dict( + FloodType = flood_type, + BasementType=bmt_type, + PostFIRM=PostFIRM, + )) return fl_config + diff --git a/pelicun/tests/dl_calculation/rulesets/MetaVarRulesets.py b/pelicun/tests/dl_calculation/rulesets/MetaVarRulesets.py index cd262c58b..447b71232 100644 --- a/pelicun/tests/dl_calculation/rulesets/MetaVarRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/MetaVarRulesets.py @@ -45,7 +45,6 @@ import numpy as np - def parse_BIM(BIM_in, location, hazards): """ Parses the information provided in the BIM model. @@ -61,8 +60,7 @@ def parse_BIM(BIM_in, location, hazards): hazard: list of str Supported hazard types: "wind", "inundation" - BIM attributes - -------------- + BIM attributes: NumberOfStories: str Number of stories YearBuilt: str @@ -101,31 +99,37 @@ def parse_BIM(BIM_in, location, hazards): if 'wind' in hazards: # maps roof type to the internal representation ap_RoofType = { - 'hip': 'hip', + 'hip' : 'hip', 'hipped': 'hip', - 'Hip': 'hip', + 'Hip' : 'hip', 'gabled': 'gab', - 'gable': 'gab', - 'Gable': 'gab', - 'flat': 'flt', - 'Flat': 'flt', + 'gable' : 'gab', + 'Gable' : 'gab', + 'flat' : 'flt', + 'Flat' : 'flt' } # maps roof system to the internal representation - ap_RoofSystem = {'Wood': 'trs', 'OWSJ': 'ows', 'N/A': 'trs'} + ap_RoofSystem = { + 'Wood': 'trs', + 'OWSJ': 'ows', + 'N/A': 'trs' + } roof_system = BIM_in.get('RoofSystem', 'Wood') - # flake8 - unused variable: `ap_NoUnits`. - # # maps number of units to the internal representation - # ap_NoUnits = { - # 'Single': 'sgl', - # 'Multiple': 'mlt', - # 'Multi': 'mlt', - # 'nav': 'nav', - # } + # maps number of units to the internal representation + ap_NoUnits = { + 'Single': 'sgl', + 'Multiple': 'mlt', + 'Multi': 'mlt', + 'nav': 'nav' + } # Average January Temp. - ap_ajt = {'Above': 'above', 'Below': 'below'} + ap_ajt = { + 'Above': 'above', + 'Below': 'below' + } # Year built alname_yearbuilt = ['yearBuilt', 'YearBuiltMODIV', 'YearBuiltNJDEP'] @@ -143,12 +147,7 @@ def parse_BIM(BIM_in, location, hazards): yearbuilt = 1985 # Number of Stories - alname_nstories = [ - 'stories', - 'NumberofStories0', - 'NumberofStories', - 'NumberofStories1', - ] + alname_nstories = ['stories', 'NumberofStories0', 'NumberofStories', 'NumberofStories1'] nstories = BIM_in.get('NumberOfStories', None) @@ -220,101 +219,95 @@ def parse_BIM(BIM_in, location, hazards): if location == 'NJ': # NJDEP code for flood zone needs to be converted buildingtype = ap_BuildingType_NJ[BIM_in['BuildingType']] - + elif location == 'LA': # standard input should provide the building type as a string buildingtype = BIM_in['BuildingType'] - # maps for design level (Marginal Engineered is mapped to - # Engineered as defauplt) - ap_DesignLevel = {'E': 'E', 'NE': 'NE', 'PE': 'PE', 'ME': 'E'} - design_level = BIM_in.get('DesignLevel', 'E') + # maps for design level (Marginal Engineered is mapped to Engineered as default) + ap_DesignLevel = { + 'E': 'E', + 'NE': 'NE', + 'PE': 'PE', + 'ME': 'E' + } + design_level = BIM_in.get('DesignLevel','E') # flood zone flood_zone = BIM_in.get('FloodZone', 'X') # add the parsed data to the BIM dict - BIM.update( - dict( - OccupancyClass=str(oc), - BuildingType=buildingtype, - YearBuilt=int(yearbuilt), - NumberOfStories=int(nstories), - PlanArea=float(area), - V_ult=float(dws), - AvgJanTemp=ap_ajt[BIM_in.get('AvgJanTemp', 'Below')], - RoofShape=ap_RoofType[BIM_in['RoofShape']], - RoofSlope=float(BIM_in.get('RoofSlope', 0.25)), # default 0.25 - SheathingThickness=float( - BIM_in.get('SheathingThick', 1.0) - ), # default 1.0 - RoofSystem=str( - ap_RoofSystem[roof_system] - ), # only valid for masonry structures - Garage=float(BIM_in.get('Garage', -1.0)), - LULC=BIM_in.get('LULC', -1), - MeanRoofHt=float(BIM_in.get('MeanRoofHt', 15.0)), # default 15 - WindowArea=float(BIM_in.get('WindowArea', 0.20)), - WindZone=str(BIM_in.get('WindZone', 'I')), - FloodZone=str(flood_zone), - ) - ) + BIM.update(dict( + OccupancyClass=str(oc), + BuildingType=buildingtype, + YearBuilt=int(yearbuilt), + NumberOfStories=int(nstories), + PlanArea=float(area), + V_ult=float(dws), + AvgJanTemp=ap_ajt[BIM_in.get('AvgJanTemp','Below')], + RoofShape=ap_RoofType[BIM_in['RoofShape']], + RoofSlope=float(BIM_in.get('RoofSlope',0.25)), # default 0.25 + SheathingThickness=float(BIM_in.get('SheathingThick',1.0)), # default 1.0 + RoofSystem=str(ap_RoofSystem[roof_system]), # only valid for masonry structures + Garage=float(BIM_in.get('Garage',-1.0)), + LULC=BIM_in.get('LULC',-1), + MeanRoofHt=float(BIM_in.get('MeanRoofHt',15.0)), # default 15 + WindowArea=float(BIM_in.get('WindowArea',0.20)), + WindZone=str(BIM_in.get('WindZone', 'I')), + FloodZone =str(flood_zone) + )) if 'inundation' in hazards: # maps for split level - ap_SplitLevel = {'NO': 0, 'YES': 1} + ap_SplitLevel = { + 'NO': 0, + 'YES': 1 + } # foundation type - foundation = BIM_in.get('FoundationType', 3501) + foundation = BIM_in.get('FoundationType',3501) # number of units - nunits = BIM_in.get('NoUnits', 1) - - # flake8 - unused variable: `ap_FloodZone`. - # # maps for flood zone - # ap_FloodZone = { - # # Coastal areas with a 1% or greater chance of flooding and an - # # additional hazard associated with storm waves. - # 6101: 'VE', - # 6102: 'VE', - # 6103: 'AE', - # 6104: 'AE', - # 6105: 'AO', - # 6106: 'AE', - # 6107: 'AH', - # 6108: 'AO', - # 6109: 'A', - # 6110: 'X', - # 6111: 'X', - # 6112: 'X', - # 6113: 'OW', - # 6114: 'D', - # 6115: 'NA', - # 6119: 'NA', - # } - - # flake8 - unused variable: `floodzone_fema`. - # if isinstance(BIM_in['FloodZone'], int): - # # NJDEP code for flood zone (conversion to the FEMA designations) - # floodzone_fema = ap_FloodZone[BIM_in['FloodZone']] - # else: - # # standard input should follow the FEMA flood zone designations - # floodzone_fema = BIM_in['FloodZone'] + nunits = BIM_in.get('NoUnits',1) + + # maps for flood zone + ap_FloodZone = { + # Coastal areas with a 1% or greater chance of flooding and an + # additional hazard associated with storm waves. + 6101: 'VE', + 6102: 'VE', + 6103: 'AE', + 6104: 'AE', + 6105: 'AO', + 6106: 'AE', + 6107: 'AH', + 6108: 'AO', + 6109: 'A', + 6110: 'X', + 6111: 'X', + 6112: 'X', + 6113: 'OW', + 6114: 'D', + 6115: 'NA', + 6119: 'NA' + } + if type(BIM_in['FloodZone']) == int: + # NJDEP code for flood zone (conversion to the FEMA designations) + floodzone_fema = ap_FloodZone[BIM_in['FloodZone']] + else: + # standard input should follow the FEMA flood zone designations + floodzone_fema = BIM_in['FloodZone'] # add the parsed data to the BIM dict - BIM.update( - dict( - DesignLevel=str(ap_DesignLevel[design_level]), # default engineered - NumberOfUnits=int(nunits), - FirstFloorElevation=float(BIM_in.get('FirstFloorHt1', 10.0)), - SplitLevel=bool( - ap_SplitLevel[BIM_in.get('SplitLevel', 'NO')] - ), # dfault: no - FoundationType=int(foundation), # default: pile - City=BIM_in.get('City', 'NA'), - ) - ) + BIM.update(dict( + DesignLevel=str(ap_DesignLevel[design_level]), # default engineered + NumberOfUnits=int(nunits), + FirstFloorElevation=float(BIM_in.get('FirstFloorHt1',10.0)), + SplitLevel=bool(ap_SplitLevel[BIM_in.get('SplitLevel','NO')]), # dfault: no + FoundationType=int(foundation), # default: pile + City=BIM_in.get('City','NA') + )) # add inferred, generic meta-variables @@ -341,12 +334,12 @@ def parse_BIM(BIM_in, location, hazards): # The flood_lim and general_lim limits depend on the year of construction if BIM['YearBuilt'] >= 2016: # In IRC 2015: - flood_lim = 130.0 # mph - general_lim = 140.0 # mph + flood_lim = 130.0 # mph + general_lim = 140.0 # mph else: # In IRC 2009 and earlier versions - flood_lim = 110.0 # mph - general_lim = 120.0 # mph + flood_lim = 110.0 # mph + general_lim = 120.0 # mph # Areas within hurricane-prone regions located in accordance with # one of the following: # (1) Within 1 mile (1.61 km) of the coastal mean high water line @@ -356,13 +349,8 @@ def parse_BIM(BIM_in, location, hazards): if not HPR: WBD = False else: - WBD = ( - ( - BIM['FloodZone'].startswith('A') - or BIM['FloodZone'].startswith('V') - ) - and BIM['V_ult'] >= flood_lim - ) or (BIM['V_ult'] >= general_lim) + WBD = (((BIM['FloodZone'].startswith('A') or BIM['FloodZone'].startswith('V')) and + BIM['V_ult'] >= flood_lim) or (BIM['V_ult'] >= general_lim)) # Terrain # open (0.03) = 3 @@ -370,94 +358,68 @@ def parse_BIM(BIM_in, location, hazards): # suburban (0.35) = 35 # light trees (0.70) = 70 # trees (1.00) = 100 - # Mapped to Land Use Categories in NJ (see - # https://www.state.nj.us/dep/gis/ - # digidownload/metadata/lulc02/anderson2002.html) by T. Wu - # group (see internal report on roughness calculations, Table - # 4). These are mapped to Hazus defintions as follows: Open - # Water (5400s) with zo=0.01 and barren land (7600) with - # zo=0.04 assume Open Open Space Developed, Low Intensity - # Developed, Medium Intensity Developed (1110-1140) assumed - # zo=0.35-0.4 assume Suburban High Intensity Developed (1600) - # with zo=0.6 assume Lt. Tree Forests of all classes - # (4100-4300) assumed zo=0.6 assume Lt. Tree Shrub (4400) with - # zo=0.06 assume Open Grasslands, pastures and agricultural - # areas (2000 series) with zo=0.1-0.15 assume Lt. Suburban - # Woody Wetlands (6250) with zo=0.3 assume suburban Emergent - # Herbaceous Wetlands (6240) with zo=0.03 assume Open - # Note: HAZUS category of trees (1.00) does not apply to any - # LU/LC in NJ - terrain = 15 # Default in Reorganized Rulesets - WIND + # Mapped to Land Use Categories in NJ (see https://www.state.nj.us/dep/gis/ + # digidownload/metadata/lulc02/anderson2002.html) by T. Wu group + # (see internal report on roughness calculations, Table 4). + # These are mapped to Hazus defintions as follows: + # Open Water (5400s) with zo=0.01 and barren land (7600) with zo=0.04 assume Open + # Open Space Developed, Low Intensity Developed, Medium Intensity Developed + # (1110-1140) assumed zo=0.35-0.4 assume Suburban + # High Intensity Developed (1600) with zo=0.6 assume Lt. Tree + # Forests of all classes (4100-4300) assumed zo=0.6 assume Lt. Tree + # Shrub (4400) with zo=0.06 assume Open + # Grasslands, pastures and agricultural areas (2000 series) with + # zo=0.1-0.15 assume Lt. Suburban + # Woody Wetlands (6250) with zo=0.3 assume suburban + # Emergent Herbaceous Wetlands (6240) with zo=0.03 assume Open + # Note: HAZUS category of trees (1.00) does not apply to any LU/LC in NJ + terrain = 15 # Default in Reorganized Rulesets - WIND if location == "NJ": - if BIM['FloodZone'].startswith('V') or BIM['FloodZone'] in [ - 'A', - 'AE', - 'A1-30', - 'AR', - 'A99', - ]: + if (BIM['FloodZone'].startswith('V') or BIM['FloodZone'] in ['A', 'AE', 'A1-30', 'AR', 'A99']): terrain = 3 - elif (BIM['LULC'] >= 5000) and (BIM['LULC'] <= 5999): - terrain = 3 # Open - elif ((BIM['LULC'] == 4400) or (BIM['LULC'] == 6240)) or ( - BIM['LULC'] == 7600 - ): - terrain = 3 # Open - elif (BIM['LULC'] >= 2000) and (BIM['LULC'] <= 2999): - terrain = 15 # Light suburban - elif ((BIM['LULC'] >= 1110) and (BIM['LULC'] <= 1140)) or ( - (BIM['LULC'] >= 6250) and (BIM['LULC'] <= 6252) - ): - terrain = 35 # Suburban - elif ((BIM['LULC'] >= 4100) and (BIM['LULC'] <= 4300)) or ( - BIM['LULC'] == 1600 - ): - terrain = 70 # light trees + elif ((BIM['LULC'] >= 5000) and (BIM['LULC'] <= 5999)): + terrain = 3 # Open + elif ((BIM['LULC'] == 4400) or (BIM['LULC'] == 6240)) or (BIM['LULC'] == 7600): + terrain = 3 # Open + elif ((BIM['LULC'] >= 2000) and (BIM['LULC'] <= 2999)): + terrain = 15 # Light suburban + elif ((BIM['LULC'] >= 1110) and (BIM['LULC'] <= 1140)) or ((BIM['LULC'] >= 6250) and (BIM['LULC'] <= 6252)): + terrain = 35 # Suburban + elif ((BIM['LULC'] >= 4100) and (BIM['LULC'] <= 4300)) or (BIM['LULC'] == 1600): + terrain = 70 # light trees elif location == "LA": - if BIM['FloodZone'].startswith('V') or BIM['FloodZone'] in [ - 'A', - 'AE', - 'A1-30', - 'AR', - 'A99', - ]: + if (BIM['FloodZone'].startswith('V') or BIM['FloodZone'] in ['A', 'AE', 'A1-30', 'AR', 'A99']): terrain = 3 - elif (BIM['LULC'] >= 50) and (BIM['LULC'] <= 59): - terrain = 3 # Open + elif ((BIM['LULC'] >= 50) and (BIM['LULC'] <= 59)): + terrain = 3 # Open elif ((BIM['LULC'] == 44) or (BIM['LULC'] == 62)) or (BIM['LULC'] == 76): - terrain = 3 # Open - elif (BIM['LULC'] >= 20) and (BIM['LULC'] <= 29): - terrain = 15 # Light suburban + terrain = 3 # Open + elif ((BIM['LULC'] >= 20) and (BIM['LULC'] <= 29)): + terrain = 15 # Light suburban elif (BIM['LULC'] == 11) or (BIM['LULC'] == 61): - terrain = 35 # Suburban - elif ((BIM['LULC'] >= 41) and (BIM['LULC'] <= 43)) or ( - BIM['LULC'] in [16, 17] - ): - terrain = 70 # light trees - - BIM.update( - dict( - # Nominal Design Wind Speed - # Former term was “Basic Wind Speed”; it is now the “Nominal Design - # Wind Speed (V_asd). Unit: mph." - V_asd=np.sqrt(0.6 * BIM['V_ult']), - HazardProneRegion=HPR, - WindBorneDebris=WBD, - TerrainRoughness=terrain, - ) - ) + terrain = 35 # Suburban + elif ((BIM['LULC'] >= 41) and (BIM['LULC'] <= 43)) or (BIM['LULC'] in [16, 17]): + terrain = 70 # light trees + + BIM.update(dict( + # Nominal Design Wind Speed + # Former term was “Basic Wind Speed”; it is now the “Nominal Design + # Wind Speed (V_asd). Unit: mph." + V_asd = np.sqrt(0.6 * BIM['V_ult']), + + HazardProneRegion=HPR, + WindBorneDebris=WBD, + TerrainRoughness=terrain, + )) if 'inundation' in hazards: - BIM.update( - dict( - # Flood Risk - # Properties in the High Water Zone (within 1 mile of - # the coast) are at risk of flooding and other - # wind-borne debris action. - # TODO: need high water zone for this and move it to inputs! - FloodRisk=True, - ) - ) + BIM.update(dict( + # Flood Risk + # Properties in the High Water Zone (within 1 mile of the coast) are at + # risk of flooding and other wind-borne debris action. + FloodRisk=True, # TODO: need high water zone for this and move it to inputs! + )) return BIM + diff --git a/pelicun/tests/dl_calculation/rulesets/WindCECBRulesets.py b/pelicun/tests/dl_calculation/rulesets/WindCECBRulesets.py index 3f732bcaa..c034a6b4f 100644 --- a/pelicun/tests/dl_calculation/rulesets/WindCECBRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/WindCECBRulesets.py @@ -44,7 +44,8 @@ # Tracy Kijewski-Correa import random - +import numpy as np +import datetime def CECB_config(BIM): """ @@ -62,7 +63,7 @@ def CECB_config(BIM): class. """ - year = BIM['YearBuilt'] # just for the sake of brevity + year = BIM['YearBuilt'] # just for the sake of brevity # Roof cover if BIM['RoofShape'] in ['gab', 'hip']: @@ -95,13 +96,14 @@ def CECB_config(BIM): # Wind Debris (widd in HAZSU) # HAZUS A: Res/Comm, B: Varies by direction, C: Residential, D: None - WIDD = 'C' # residential (default) - if BIM['OccupancyClass'] in ['RES1', 'RES2', 'RES3A', 'RES3B', 'RES3C', 'RES3D']: - WIDD = 'C' # residential + WIDD = 'C' # residential (default) + if BIM['OccupancyClass'] in ['RES1', 'RES2', 'RES3A', 'RES3B', 'RES3C', + 'RES3D']: + WIDD = 'C' # residential elif BIM['OccupancyClass'] == 'AGR1': - WIDD = 'D' # None + WIDD = 'D' # None else: - WIDD = 'A' # Res/Comm + WIDD = 'A' # Res/Comm # Window area ratio if BIM['WindowArea'] < 0.33: @@ -119,22 +121,19 @@ def CECB_config(BIM): bldg_tag = 'C.ECB.H' # extend the BIM dictionary - BIM.update( - dict( - RoofCover=roof_cover, - Shutters=shutters, - WindowAreaRatio=WWR, - WindDebrisClass=WIDD, - ) - ) + BIM.update(dict( + RoofCover = roof_cover, + Shutters = shutters, + WindowAreaRatio = WWR, + WindDebrisClass = WIDD + )) - bldg_config = ( - f"{bldg_tag}." - f"{roof_cover}." - f"{int(shutters)}." - f"{WIDD}." - f"{WWR}." - f"{int(BIM['TerrainRoughness'])}" - ) + bldg_config = f"{bldg_tag}." \ + f"{roof_cover}." \ + f"{int(shutters)}." \ + f"{WIDD}." \ + f"{WWR}." \ + f"{int(BIM['TerrainRoughness'])}" return bldg_config + diff --git a/pelicun/tests/dl_calculation/rulesets/WindCERBRulesets.py b/pelicun/tests/dl_calculation/rulesets/WindCERBRulesets.py index eba699a6d..41f8faab0 100644 --- a/pelicun/tests/dl_calculation/rulesets/WindCERBRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/WindCERBRulesets.py @@ -44,7 +44,8 @@ # Tracy Kijewski-Correa import random - +import numpy as np +import datetime def CERB_config(BIM): """ @@ -62,7 +63,7 @@ def CERB_config(BIM): class. """ - year = BIM['YearBuilt'] # just for the sake of brevity + year = BIM['YearBuilt'] # just for the sake of brevity # Roof cover if BIM['RoofShape'] in ['gab', 'hip']: @@ -95,13 +96,14 @@ def CERB_config(BIM): # Wind Debris (widd in HAZUS) # HAZUS A: Res/Comm, B: Varies by direction, C: Residential, D: None - WIDD = 'C' # residential (default) - if BIM['OccupancyClass'] in ['RES1', 'RES2', 'RES3A', 'RES3B', 'RES3C', 'RES3D']: - WIDD = 'C' # residential + WIDD = 'C' # residential (default) + if BIM['OccupancyClass'] in ['RES1', 'RES2', 'RES3A', 'RES3B', 'RES3C', + 'RES3D']: + WIDD = 'C' # residential elif BIM['OccupancyClass'] == 'AGR1': - WIDD = 'D' # None + WIDD = 'D' # None else: - WIDD = 'A' # Res/Comm + WIDD = 'A' # Res/Comm # Window area ratio if BIM['WindowArea'] < 0.33: @@ -119,22 +121,18 @@ def CERB_config(BIM): bldg_tag = 'C.ERB.H' # extend the BIM dictionary - BIM.update( - dict( - RoofCover=roof_cover, - Shutters=shutters, - WindowAreaRatio=WWR, - WindDebrisClass=WIDD, - ) - ) + BIM.update(dict( + RoofCover = roof_cover, + Shutters = shutters, + WindowAreaRatio = WWR, + WindDebrisClass = WIDD + )) - bldg_config = ( - f"{bldg_tag}." - f"{roof_cover}." - f"{int(shutters)}." - f"{WIDD}." - f"{WWR}." - f"{int(BIM['TerrainRoughness'])}" - ) + bldg_config = f"{bldg_tag}." \ + f"{roof_cover}." \ + f"{int(shutters)}." \ + f"{WIDD}." \ + f"{WWR}." \ + f"{int(BIM['TerrainRoughness'])}" return bldg_config diff --git a/pelicun/tests/dl_calculation/rulesets/WindEFRulesets.py b/pelicun/tests/dl_calculation/rulesets/WindEFRulesets.py index 9e3245457..1762eb5ce 100644 --- a/pelicun/tests/dl_calculation/rulesets/WindEFRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/WindEFRulesets.py @@ -44,6 +44,7 @@ # Tracy Kijewski-Correa import random +import numpy as np import datetime @@ -77,9 +78,9 @@ def HUEFFS_config(BIM): # Roof deck age if year >= (datetime.datetime.now().year - 50): - DQ = 'god' # new or average + DQ = 'god' # new or average else: - DQ = 'por' # old + DQ = 'por' # old # Metal-RDA if year > 2000: @@ -94,30 +95,25 @@ def HUEFFS_config(BIM): shutters = int(BIM['WBD']) # extend the BIM dictionary - BIM.update( - dict( - RoofCover=roof_cover, - RoofDeckAttachmentM=MRDA, - RoofDeckAge=DQ, - WindDebrisClass=WIDD, - Shutters=shutters, - ) - ) + BIM.update(dict( + RoofCover = roof_cover, + RoofDeckAttachmentM = MRDA, + RoofDeckAge=DQ, + WindDebrisClass = WIDD, + Shutters = shutters + )) bldg_tag = 'HUEF.FS' - bldg_config = ( - f"{bldg_tag}." - f"{roof_cover}." - f"{shutters}." - f"{WIDD}." - f"{DQ}." - f"{MRDA}." - f"{int(BIM['TerrainRoughness'])}" - ) + bldg_config = f"{bldg_tag}." \ + f"{roof_cover}." \ + f"{shutters}." \ + f"{WIDD}." \ + f"{DQ}." \ + f"{MRDA}." \ + f"{int(BIM['TerrainRoughness'])}" return bldg_config - def HUEFSS_config(BIM): """ Rules to identify a HAZUS HUEFFS/HUEFSS configuration based on BIM data @@ -148,9 +144,9 @@ def HUEFSS_config(BIM): # Roof deck age if year >= (datetime.datetime.now().year - 50): - DQ = 'god' # new or average + DQ = 'god' # new or average else: - DQ = 'por' # old + DQ = 'por' # old # Metal-RDA if year > 2000: @@ -165,26 +161,22 @@ def HUEFSS_config(BIM): shutters = BIM['WindBorneDebris'] # extend the BIM dictionary - BIM.update( - dict( - RoofCover=roof_cover, - RoofDeckAttachmentM=MRDA, - RoofDeckAge=DQ, - WindDebrisClass=WIDD, - Shutters=shutters, - ) - ) + BIM.update(dict( + RoofCover = roof_cover, + RoofDeckAttachmentM = MRDA, + RoofDeckAge=DQ, + WindDebrisClass = WIDD, + Shutters=shutters + )) bldg_tag = 'HUEF.S.S' - bldg_config = ( - f"{bldg_tag}." - f"{roof_cover}." - f"{int(shutters)}." - f"{WIDD}." - f"{DQ}." - f"{MRDA}." - f"{int(BIM['TerrainRoughness'])}" - ) + bldg_config = f"{bldg_tag}." \ + f"{roof_cover}." \ + f"{int(shutters)}." \ + f"{WIDD}." \ + f"{DQ}." \ + f"{MRDA}." \ + f"{int(BIM['TerrainRoughness'])}" return bldg_config @@ -229,7 +221,7 @@ def HUEFH_config(BIM): else: MRDA = 'std' # standard - if BIM['NumberOfStories'] <= 2: + if BIM['NumberOfStories'] <=2: bldg_tag = 'HUEF.H.S' elif BIM['NumberOfStories'] <= 5: bldg_tag = 'HUEF.H.M' @@ -237,27 +229,22 @@ def HUEFH_config(BIM): bldg_tag = 'HUEF.H.L' # extend the BIM dictionary - BIM.update( - dict( - RoofCover=roof_cover, - RoofDeckAttachmentM=MRDA, - WindDebrisClass=WIDD, - Shutters=shutters, - ) - ) - - bldg_config = ( - f"{bldg_tag}." - f"{roof_cover}." - f"{WIDD}." - f"{MRDA}." - f"{int(shutters)}." - f"{int(BIM['TerrainRoughness'])}" - ) + BIM.update(dict( + RoofCover = roof_cover, + RoofDeckAttachmentM = MRDA, + WindDebrisClass = WIDD, + Shutters=shutters + )) + + bldg_config = f"{bldg_tag}." \ + f"{roof_cover}." \ + f"{WIDD}." \ + f"{MRDA}." \ + f"{int(shutters)}." \ + f"{int(BIM['TerrainRoughness'])}" return bldg_config - def HUEFS_config(BIM): """ Rules to identify a HAZUS HUEFS configuration based on BIM data @@ -305,29 +292,25 @@ def HUEFS_config(BIM): else: MRDA = 'std' # standard - if BIM['NumberOfStories'] <= 2: + if BIM['NumberOfStories'] <=2: bldg_tag = 'HUEF.S.M' else: bldg_tag = 'HUEF.S.L' # extend the BIM dictionary - BIM.update( - dict( - RoofCover=roof_cover, - RoofDeckAttachmentM=MRDA, - WindDebrisClass=WIDD, - Shutters=shutters, - ) - ) - - bldg_config = ( - f"{bldg_tag}." - f"{roof_cover}." - f"{int(shutters)}." - f"{WIDD}." - f"null." - f"{MRDA}." - f"{int(BIM['TerrainRoughness'])}" - ) - - return bldg_config + BIM.update(dict( + RoofCover = roof_cover, + RoofDeckAttachmentM = MRDA, + WindDebrisClass = WIDD, + Shutters=shutters + )) + + bldg_config = f"{bldg_tag}." \ + f"{roof_cover}." \ + f"{int(shutters)}." \ + f"{WIDD}." \ + f"null." \ + f"{MRDA}." \ + f"{int(BIM['TerrainRoughness'])}" + + return bldg_config \ No newline at end of file diff --git a/pelicun/tests/dl_calculation/rulesets/WindMECBRulesets.py b/pelicun/tests/dl_calculation/rulesets/WindMECBRulesets.py index 02e34a1be..137844f7b 100644 --- a/pelicun/tests/dl_calculation/rulesets/WindMECBRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/WindMECBRulesets.py @@ -45,7 +45,6 @@ import random - def MECB_config(BIM): """ Rules to identify a HAZUS MECB configuration based on BIM data @@ -62,7 +61,7 @@ def MECB_config(BIM): class. """ - year = BIM['YearBuilt'] # just for the sake of brevity + year = BIM['YearBuilt'] # just for the sake of brevity # Roof cover if BIM['RoofShape'] in ['gab', 'hip']: @@ -86,13 +85,14 @@ def MECB_config(BIM): # Wind Debris (widd in HAZSU) # HAZUS A: Res/Comm, B: Varies by direction, C: Residential, D: None - WIDD = 'C' # residential (default) - if BIM['OccupancyClass'] in ['RES1', 'RES2', 'RES3A', 'RES3B', 'RES3C', 'RES3D']: - WIDD = 'C' # residential + WIDD = 'C' # residential (default) + if BIM['OccupancyClass'] in ['RES1', 'RES2', 'RES3A', 'RES3B', 'RES3C', + 'RES3D']: + WIDD = 'C' # residential elif BIM['OccupancyClass'] == 'AGR1': - WIDD = 'D' # None + WIDD = 'D' # None else: - WIDD = 'A' # Res/Comm + WIDD = 'A' # Res/Comm # Metal RDA # 1507.2.8.1 High Wind Attachment. @@ -122,24 +122,20 @@ def MECB_config(BIM): bldg_tag = 'M.ECB.H' # extend the BIM dictionary - BIM.update( - dict( - RoofCover=roof_cover, - RoofDeckAttachmentM=MRDA, - Shutters=shutters, - WindowAreaRatio=WWR, - WindDebrisClass=WIDD, - ) - ) + BIM.update(dict( + RoofCover = roof_cover, + RoofDeckAttachmentM = MRDA, + Shutters = shutters, + WindowAreaRatio = WWR, + WindDebrisClass = WIDD + )) - bldg_config = ( - f"{bldg_tag}." - f"{roof_cover}." - f"{int(shutters)}." - f"{WIDD}." - f"{MRDA}." - f"{WWR}." - f"{int(BIM['TerrainRoughness'])}" - ) + bldg_config = f"{bldg_tag}." \ + f"{roof_cover}." \ + f"{int(shutters)}." \ + f"{WIDD}." \ + f"{MRDA}." \ + f"{WWR}." \ + f"{int(BIM['TerrainRoughness'])}" return bldg_config diff --git a/pelicun/tests/dl_calculation/rulesets/WindMERBRulesets.py b/pelicun/tests/dl_calculation/rulesets/WindMERBRulesets.py index ab015762e..2299b8dbb 100644 --- a/pelicun/tests/dl_calculation/rulesets/WindMERBRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/WindMERBRulesets.py @@ -44,7 +44,8 @@ # Tracy Kijewski-Correa import random - +import numpy as np +import datetime def MERB_config(BIM): """ @@ -62,7 +63,7 @@ def MERB_config(BIM): class. """ - year = BIM['YearBuilt'] # just for the sake of brevity + year = BIM['YearBuilt'] # just for the sake of brevity # Roof cover if BIM['RoofShape'] in ['gab', 'hip']: @@ -86,13 +87,14 @@ def MERB_config(BIM): # Wind Debris (widd in HAZSU) # HAZUS A: Res/Comm, B: Varies by direction, C: Residential, D: None - WIDD = 'C' # residential (default) - if BIM['OccupancyClass'] in ['RES1', 'RES2', 'RES3A', 'RES3B', 'RES3C', 'RES3D']: - WIDD = 'C' # residential + WIDD = 'C' # residential (default) + if BIM['OccupancyClass'] in ['RES1', 'RES2', 'RES3A', 'RES3B', 'RES3C', + 'RES3D']: + WIDD = 'C' # residential elif BIM['OccupancyClass'] == 'AGR1': - WIDD = 'D' # None + WIDD = 'D' # None else: - WIDD = 'A' # Res/Comm + WIDD = 'A' # Res/Comm # Metal RDA # 1507.2.8.1 High Wind Attachment. @@ -122,24 +124,20 @@ def MERB_config(BIM): bldg_tag = 'M.ERB.H' # extend the BIM dictionary - BIM.update( - dict( - RoofCover=roof_cover, - RoofDeckAttachmentM=MRDA, - Shutters=shutters, - WindowAreaRatio=WWR, - WindDebrisClass=WIDD, - ) - ) + BIM.update(dict( + RoofCover = roof_cover, + RoofDeckAttachmentM = MRDA, + Shutters = shutters, + WindowAreaRatio = WWR, + WindDebrisClass = WIDD + )) - bldg_config = ( - f"{bldg_tag}." - f"{roof_cover}." - f"{int(shutters)}." - f"{WIDD}." - f"{MRDA}." - f"{WWR}." - f"{int(BIM['TerrainRoughness'])}" - ) + bldg_config = f"{bldg_tag}." \ + f"{roof_cover}." \ + f"{int(shutters)}." \ + f"{WIDD}." \ + f"{MRDA}." \ + f"{WWR}." \ + f"{int(BIM['TerrainRoughness'])}" return bldg_config diff --git a/pelicun/tests/dl_calculation/rulesets/WindMHRulesets.py b/pelicun/tests/dl_calculation/rulesets/WindMHRulesets.py index d92004de9..db6ebe8a3 100644 --- a/pelicun/tests/dl_calculation/rulesets/WindMHRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/WindMHRulesets.py @@ -44,7 +44,8 @@ # Tracy Kijewski-Correa import random - +import numpy as np +import datetime def MH_config(BIM): """ @@ -62,7 +63,7 @@ def MH_config(BIM): class. """ - year = BIM['YearBuilt'] # just for the sake of brevity + year = BIM['YearBuilt'] # just for the sake of brevity if year <= 1976: # MHPHUD bldg_tag = 'MH.PHUD' @@ -98,18 +99,15 @@ def MH_config(BIM): bldg_tag = 'MH.94HUD' + BIM['WindZone'] # extend the BIM dictionary - BIM.update( - dict( - TieDowns=TD, - Shutters=shutters, - ) - ) + BIM.update(dict( + TieDowns = TD, + Shutters = shutters, + )) - bldg_config = ( - f"{bldg_tag}." - f"{int(shutters)}." - f"{int(TD)}." - f"{int(BIM['TerrainRoughness'])}" - ) + bldg_config = f"{bldg_tag}." \ + f"{int(shutters)}." \ + f"{int(TD)}." \ + f"{int(BIM['TerrainRoughness'])}" return bldg_config + diff --git a/pelicun/tests/dl_calculation/rulesets/WindMLRIRulesets.py b/pelicun/tests/dl_calculation/rulesets/WindMLRIRulesets.py index a09c56cdf..09b833976 100644 --- a/pelicun/tests/dl_calculation/rulesets/WindMLRIRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/WindMLRIRulesets.py @@ -43,9 +43,10 @@ # Meredith Lockhead # Tracy Kijewski-Correa +import random +import numpy as np import datetime - def MLRI_config(BIM): """ Rules to identify a HAZUS MLRI configuration based on BIM data @@ -62,7 +63,7 @@ def MLRI_config(BIM): class. """ - year = BIM['YearBuilt'] # just for the sake of brevity + year = BIM['YearBuilt'] # just for the sake of brevity # MR MR = True @@ -84,7 +85,7 @@ def MLRI_config(BIM): if BIM['RoofShape'] in ['gab', 'hip']: roof_cover = 'null' - roof_quality = 'god' # default supported by HAZUS + roof_quality = 'god' # default supported by HAZUS else: if year >= 1975: roof_cover = 'spm' @@ -99,25 +100,22 @@ def MLRI_config(BIM): roof_quality = 'god' else: roof_quality = 'por' - + # extend the BIM dictionary - BIM.update( - dict( - RoofCover=roof_cover, - RoofQuality=roof_quality, - RoofDeckAttachmentM=MRDA, - Shutters=shutters, - MasonryReinforcing=MR, - ) - ) + BIM.update(dict( + RoofCover = roof_cover, + RoofQuality = roof_quality, + RoofDeckAttachmentM = MRDA, + Shutters = shutters, + MasonryReinforcing = MR, + )) - bldg_config = ( - f"M.LRI." - f"{int(shutters)}." - f"{int(MR)}." - f"{roof_quality}." - f"{MRDA}." - f"{int(BIM['TerrainRoughness'])}" - ) + bldg_config = f"M.LRI." \ + f"{int(shutters)}." \ + f"{int(MR)}." \ + f"{roof_quality}." \ + f"{MRDA}." \ + f"{int(BIM['TerrainRoughness'])}" return bldg_config + diff --git a/pelicun/tests/dl_calculation/rulesets/WindMLRMRulesets.py b/pelicun/tests/dl_calculation/rulesets/WindMLRMRulesets.py index 3b5e6246e..c63f39313 100644 --- a/pelicun/tests/dl_calculation/rulesets/WindMLRMRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/WindMLRMRulesets.py @@ -44,9 +44,9 @@ # Tracy Kijewski-Correa import random +import numpy as np import datetime - def MLRM_config(BIM): """ Rules to identify a HAZUS MLRM configuration based on BIM data @@ -63,7 +63,7 @@ def MLRM_config(BIM): class. """ - year = BIM['YearBuilt'] # just for the sake of brevity + year = BIM['YearBuilt'] # just for the sake of brevity # Note the only roof option for commercial masonry in NJ appraisers manual # is OSWJ, so this suggests they do not even see alternate roof system @@ -81,41 +81,40 @@ def MLRM_config(BIM): # Shutters # IRC 2000-2015: - # R301.2.1.2 in NJ IRC 2015 says protection of openings required - # for buildings located in WindBorneDebris regions, mentions - # impact-rated protection for glazing, impact-resistance for - # garage door glazed openings, and finally states that wood - # structural panels with a thickness > 7/16" and a span <8' can be - # used, as long as they are precut, attached to the framing - # surrounding the opening, and the attachments are resistant to - # corrosion and are able to resist component and cladding loads; + # R301.2.1.2 in NJ IRC 2015 says protection of openings required for + # buildings located in WindBorneDebris regions, mentions impact-rated protection for + # glazing, impact-resistance for garage door glazed openings, and finally + # states that wood structural panels with a thickness > 7/16" and a + # span <8' can be used, as long as they are precut, attached to the framing + # surrounding the opening, and the attachments are resistant to corrosion + # and are able to resist component and cladding loads; # Earlier IRC editions provide similar rules. shutters = BIM['WindBorneDebris'] # Masonry Reinforcing (MR) - # R606.6.4.1.2 Metal Reinforcement states that walls other than - # interior non-load-bearing walls shall be anchored at vertical - # intervals of not more than 8 inches with joint reinforcement of - # not less than 9 gage. Therefore this ruleset assumes that all - # exterior or load-bearing masonry walls will have - # reinforcement. Since our considerations deal with wind speed, I - # made the assumption that only exterior walls are being taken + # R606.6.4.1.2 Metal Reinforcement states that walls other than interior + # non-load-bearing walls shall be anchored at vertical intervals of not + # more than 8 inches with joint reinforcement of not less than 9 gage. + # Therefore this ruleset assumes that all exterior or load-bearing masonry + # walls will have reinforcement. Since our considerations deal with wind + # speed, I made the assumption that only exterior walls are being taken # into consideration. MR = True # Wind Debris (widd in HAZSU) # HAZUS A: Res/Comm, B: Varies by direction, C: Residential, D: None - WIDD = 'C' # residential (default) - if BIM['OccupancyClass'] in ['RES1', 'RES2', 'RES3A', 'RES3B', 'RES3C', 'RES3D']: - WIDD = 'C' # residential + WIDD = 'C' # residential (default) + if BIM['OccupancyClass'] in ['RES1', 'RES2', 'RES3A', 'RES3B', 'RES3C', + 'RES3D']: + WIDD = 'C' # residential elif BIM['OccupancyClass'] == 'AGR1': - WIDD = 'D' # None + WIDD = 'D' # None else: - WIDD = 'A' # Res/Comm + WIDD = 'A' # Res/Comm if BIM['RoofSystem'] == 'ows': # RDA - RDA = 'null' # Doesn't apply to OWSJ + RDA = 'null' # Doesn't apply to OWSJ # Roof deck age (DQ) # Average lifespan of a steel joist roof is roughly 50 years according @@ -123,9 +122,9 @@ def MLRM_config(BIM): # current year, the roof deck should be considered old. # https://www.metalroofing.systems/metal-roofing-pros-cons/ if year >= (datetime.datetime.now().year - 50): - DQ = 'god' # new or average + DQ = 'god' # new or average else: - DQ = 'por' # old + DQ = 'por' # old # RWC RWC = 'null' # Doesn't apply to OWSJ @@ -145,7 +144,7 @@ def MLRM_config(BIM): elif BIM['RoofSystem'] == 'trs': # This clause should not be activated for NJ # RDA - if BIM['TerrainRoughness'] >= 35: # suburban or light trees + if BIM['TerrainRoughness'] >= 35: # suburban or light trees if BIM['V_ult'] > 130.0: RDA = '8s' # 8d @ 6"/6" 'D' else: @@ -157,10 +156,10 @@ def MLRM_config(BIM): RDA = '8d' # 8d @ 6"/12" 'B' # Metal RDA - MRDA = 'null' # Doesn't apply to Wood Truss + MRDA = 'null' # Doesn't apply to Wood Truss # Roof deck agea (DQ) - DQ = 'null' # Doesn't apply to Wood Truss + DQ = 'null' # Doesn't apply to Wood Truss # RWC if BIM['V_ult'] > 110: @@ -179,33 +178,29 @@ def MLRM_config(BIM): if BIM['MeanRoofHt'] < 15.0: # extend the BIM dictionary - BIM.update( - dict( - RoofCover=roof_cover, - RoofDeckAttachmentW=RDA, - RoofDeckAttachmentM=MRDA, - RoofDeckAge=DQ, - RoofToWallConnection=RWC, - Shutters=shutters, - MasonryReinforcing=MR, - WindowAreaRatio=WIDD, - ) - ) + BIM.update(dict( + RoofCover = roof_cover, + RoofDeckAttachmentW = RDA, + RoofDeckAttachmentM = MRDA, + RoofDeckAge = DQ, + RoofToWallConnection = RWC, + Shutters = shutters, + MasonryReinforcing = MR, + WindowAreaRatio = WIDD + )) # if it's MLRM1, configure outputs - bldg_config = ( - f"M.LRM.1." - f"{roof_cover}." - f"{int(shutters)}." - f"{int(MR)}." - f"{WIDD}." - f"{BIM['RoofSystem']}." - f"{RDA}." - f"{RWC}." - f"{DQ}." - f"{MRDA}." - f"{int(BIM['TerrainRoughness'])}" - ) + bldg_config = f"M.LRM.1." \ + f"{roof_cover}." \ + f"{int(shutters)}." \ + f"{int(MR)}." \ + f"{WIDD}." \ + f"{BIM['RoofSystem']}." \ + f"{RDA}." \ + f"{RWC}." \ + f"{DQ}." \ + f"{MRDA}." \ + f"{int(BIM['TerrainRoughness'])}" else: unit_tag = 'null' @@ -222,34 +217,30 @@ def MLRM_config(BIM): unit_tag = 'mlt' # extend the BIM dictionary - BIM.update( - dict( - RoofCover=roof_cover, - RoofDeckAttachmentW=RDA, - RoofDeckAttachmentM=MRDA, - RoofDeckAge=DQ, - RoofToWallConnection=RWC, - Shutters=shutters, - MasonryReinforcing=MR, - WindDebrisClass=WIDD, - UnitType=unit_tag, - ) - ) - - bldg_config = ( - f"M.LRM.2." - f"{roof_cover}." - f"{int(shutters)}." - f"{int(MR)}." - f"{WIDD}." - f"{BIM['RoofSystem']}." - f"{RDA}." - f"{RWC}." - f"{DQ}." - f"{MRDA}." - f"{unit_tag}." - f"{joist_spacing}." - f"{int(BIM['TerrainRoughness'])}" - ) - - return bldg_config + BIM.update(dict( + RoofCover = roof_cover, + RoofDeckAttachmentW = RDA, + RoofDeckAttachmentM = MRDA, + RoofDeckAge = DQ, + RoofToWallConnection = RWC, + Shutters = shutters, + MasonryReinforcing = MR, + WindDebrisClass = WIDD, + UnitType=unit_tag + )) + + bldg_config = f"M.LRM.2." \ + f"{roof_cover}." \ + f"{int(shutters)}." \ + f"{int(MR)}." \ + f"{WIDD}." \ + f"{BIM['RoofSystem']}." \ + f"{RDA}." \ + f"{RWC}." \ + f"{DQ}." \ + f"{MRDA}." \ + f"{unit_tag}." \ + f"{joist_spacing}." \ + f"{int(BIM['TerrainRoughness'])}" + + return bldg_config \ No newline at end of file diff --git a/pelicun/tests/dl_calculation/rulesets/WindMMUHRulesets.py b/pelicun/tests/dl_calculation/rulesets/WindMMUHRulesets.py index 556dfe16d..3d27cbe09 100644 --- a/pelicun/tests/dl_calculation/rulesets/WindMMUHRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/WindMMUHRulesets.py @@ -46,7 +46,6 @@ import random import datetime - def MMUH_config(BIM): """ Rules to identify a HAZUS MMUH configuration based on BIM data @@ -63,13 +62,13 @@ def MMUH_config(BIM): class. """ - year = BIM['YearBuilt'] # just for the sake of brevity + year = BIM['YearBuilt'] # just for the sake of brevity # Secondary Water Resistance (SWR) # Minimum drainage recommendations are in place in NJ (See below). # However, SWR indicates a code-plus practice. - SWR = "null" # Default + SWR = "null" # Default if BIM['RoofShape'] == 'flt': SWR = 'null' elif BIM['RoofShape'] in ['hip', 'gab']: @@ -128,7 +127,7 @@ def MMUH_config(BIM): # roughness length in the ruleset herein. # The base rule was then extended to the exposures closest to suburban and # light suburban, even though these are not considered by the code. - if BIM['TerrainRoughness'] >= 35: # suburban or light trees + if BIM['TerrainRoughness'] >= 35: # suburban or light trees if BIM['V_ult'] > 130.0: RDA = '8s' # 8d @ 6"/6" 'D' else: @@ -147,14 +146,13 @@ def MMUH_config(BIM): # Shutters # IRC 2000-2015: - # R301.2.1.2 in NJ IRC 2015 says protection of openings required - # for buildings located in WindBorneDebris regions, mentions - # impact-rated protection for glazing, impact-resistance for - # garage door glazed openings, and finally states that wood - # structural panels with a thickness > 7/16" and a span <8' can be - # used, as long as they are precut, attached to the framing - # surrounding the opening, and the attachments are resistant to - # corrosion and are able to resist component and cladding loads; + # R301.2.1.2 in NJ IRC 2015 says protection of openings required for + # buildings located in WindBorneDebris regions, mentions impact-rated protection for + # glazing, impact-resistance for garage door glazed openings, and finally + # states that wood structural panels with a thickness > 7/16" and a + # span <8' can be used, as long as they are precut, attached to the framing + # surrounding the opening, and the attachments are resistant to corrosion + # and are able to resist component and cladding loads; # Earlier IRC editions provide similar rules. if year >= 2000: shutters = BIM['WindBorneDebris'] @@ -186,30 +184,26 @@ def MMUH_config(BIM): stories = min(BIM['NumberOfStories'], 3) # extend the BIM dictionary - BIM.update( - dict( - SecondaryWaterResistance=SWR, - RoofCover=roof_cover, - RoofQuality=roof_quality, - RoofDeckAttachmentW=RDA, - RoofToWallConnection=RWC, - Shutters=shutters, - MasonryReinforcing=MR, - ) - ) - - bldg_config = ( - f"M.MUH." - f"{int(stories)}." - f"{BIM['RoofShape']}." - f"{int(SWR)}." - f"{roof_cover}." - f"{roof_quality}." - f"{RDA}." - f"{RWC}." - f"{int(shutters)}." - f"{int(MR)}." - f"{int(BIM['TerrainRoughness'])}" - ) + BIM.update(dict( + SecondaryWaterResistance = SWR, + RoofCover = roof_cover, + RoofQuality = roof_quality, + RoofDeckAttachmentW = RDA, + RoofToWallConnection = RWC, + Shutters = shutters, + MasonryReinforcing = MR, + )) + + bldg_config = f"M.MUH." \ + f"{int(stories)}." \ + f"{BIM['RoofShape']}." \ + f"{int(SWR)}." \ + f"{roof_cover}." \ + f"{roof_quality}." \ + f"{RDA}." \ + f"{RWC}." \ + f"{int(shutters)}." \ + f"{int(MR)}." \ + f"{int(BIM['TerrainRoughness'])}" return bldg_config diff --git a/pelicun/tests/dl_calculation/rulesets/WindMSFRulesets.py b/pelicun/tests/dl_calculation/rulesets/WindMSFRulesets.py index b8c8e1fbd..a9878d9de 100644 --- a/pelicun/tests/dl_calculation/rulesets/WindMSFRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/WindMSFRulesets.py @@ -46,7 +46,6 @@ import random import datetime - def MSF_config(BIM): """ Rules to identify a HAZUS MSF configuration based on BIM data @@ -63,7 +62,7 @@ def MSF_config(BIM): class. """ - year = BIM['YearBuilt'] # just for the sake of brevity + year = BIM['YearBuilt'] # just for the sake of brevity # Roof-Wall Connection (RWC) if BIM['HazardProneRegion']: @@ -79,14 +78,13 @@ def MSF_config(BIM): # Shutters # IRC 2000-2015: - # R301.2.1.2 in NJ IRC 2015 says protection of openings required - # for buildings located in WindBorneDebris regions, mentions - # impact-rated protection for glazing, impact-resistance for - # garage door glazed openings, and finally states that wood - # structural panels with a thickness > 7/16" and a span <8' can be - # used, as long as they are precut, attached to the framing - # surrounding the opening, and the attachments are resistant to - # corrosion and are able to resist component and cladding loads; + # R301.2.1.2 in NJ IRC 2015 says protection of openings required for + # buildings located in WindBorneDebris regions, mentions impact-rated protection for + # glazing, impact-resistance for garage door glazed openings, and finally + # states that wood structural panels with a thickness > 7/16" and a + # span <8' can be used, as long as they are precut, attached to the framing + # surrounding the opening, and the attachments are resistant to corrosion + # and are able to resist component and cladding loads; # Earlier IRC editions provide similar rules. if year >= 2000: shutters = BIM['WindBorneDebris'] @@ -105,6 +103,7 @@ def MSF_config(BIM): else: shutters = False + if BIM['RoofSystem'] == 'trs': # Roof Deck Attachment (RDA) @@ -115,13 +114,13 @@ def MSF_config(BIM): # codes. Commentary for Table R602.3(1) indicates 8d nails with 6”/6” # spacing (enhanced roof spacing) for ultimate wind speeds greater than # a speed_lim. speed_lim depends on the year of construction - RDA = '6d' # Default (aka A) in Reorganized Rulesets - WIND + RDA = '6d' # Default (aka A) in Reorganized Rulesets - WIND if year >= 2016: # IRC 2015 - speed_lim = 130.0 # mph + speed_lim = 130.0 # mph else: # IRC 2000 - 2009 - speed_lim = 100.0 # mph + speed_lim = 100.0 # mph if BIM['V_ult'] > speed_lim: RDA = '8s' # 8d @ 6"/6" ('D' in the Reorganized Rulesets - WIND) else: @@ -153,21 +152,21 @@ def MSF_config(BIM): else: if year > (datetime.datetime.now().year - 30): if BIM['Garage'] < 1: - garage = 'no' # None + garage = 'no' # None else: if shutters: - garage = 'sup' # SFBC 1994 + garage = 'sup' # SFBC 1994 else: - garage = 'std' # Standard + garage = 'std' # Standard else: # year <= current year - 30 if BIM['Garage'] < 1: - garage = 'no' # None + garage = 'no' # None else: if shutters: garage = 'sup' else: - garage = 'wkd' # Weak + garage = 'wkd' # Weak # Masonry Reinforcing (MR) # R606.6.4.1.2 Metal Reinforcement states that walls other than interior @@ -182,31 +181,27 @@ def MSF_config(BIM): stories = min(BIM['NumberOfStories'], 2) # extend the BIM dictionary - BIM.update( - dict( - SecondaryWaterResistance=SWR, - RoofDeckAttachmentW=RDA, - RoofToWallConnection=RWC, - Shutters=shutters, - AugmentGarage=garage, - MasonryReinforcing=MR, - ) - ) + BIM.update(dict( + SecondaryWaterResistance = SWR, + RoofDeckAttachmentW = RDA, + RoofToWallConnection = RWC, + Shutters = shutters, + AugmentGarage = garage, + MasonryReinforcing = MR, + )) - bldg_config = ( - f"M.SF." - f"{int(stories)}." - f"{BIM['RoofShape']}." - f"{RWC}." - f"{RFT}." - f"{RDA}." - f"{int(shutters)}." - f"{int(SWR)}." - f"{garage}." - f"{int(MR)}." - f"null." - f"{int(BIM['TerrainRoughness'])}" - ) + bldg_config = f"M.SF." \ + f"{int(stories)}." \ + f"{BIM['RoofShape']}." \ + f"{RWC}." \ + f"{RFT}." \ + f"{RDA}." \ + f"{int(shutters)}." \ + f"{int(SWR)}." \ + f"{garage}." \ + f"{int(MR)}." \ + f"null." \ + f"{int(BIM['TerrainRoughness'])}" else: # Roof system = OSJW @@ -225,50 +220,43 @@ def MSF_config(BIM): # NJ IBC 1507.4.5 (for smtl) # high wind attachment are required for DSWII > 142 mph if BIM['V_ult'] > 142.0: - RDA = 'sup' # superior + RDA = 'sup' # superior else: - RDA = 'std' # standard + RDA = 'std' # standard # Secondary Water Resistance (SWR) # Minimum drainage recommendations are in place in NJ (See below). # However, SWR indicates a code-plus practice. - SWR = 'null' # Default + SWR = 'null' # Default if BIM['RoofShape'] == 'flt': SWR = int(True) - elif ( - (BIM['RoofShape'] in ['hip', 'gab']) - and (roof_cover == 'cshl') - and (RDA == 'sup') - ): + elif ((BIM['RoofShape'] in ['hip', 'gab']) and + (roof_cover=='cshl') and (RDA=='sup')): SWR = int(random.random() < 0.6) stories = min(BIM['NumberOfStories'], 2) # extend the BIM dictionary - BIM.update( - dict( - SecondaryWaterResistance=SWR, - RoofDeckAttachmentW=RDA, - RoofToWallConnection=RWC, - Shutters=shutters, - AugmentGarage=garage, - MasonryReinforcing=MR, - ) - ) + BIM.update(dict( + SecondaryWaterResistance = SWR, + RoofDeckAttachmentW = RDA, + RoofToWallConnection = RWC, + Shutters = shutters, + AugmentGarage = garage, + MasonryReinforcing = MR, + )) - bldg_config = ( - f"M.SF." - f"{int(stories)}." - f"{BIM['RoofShape']}." - f"{RWC}." - f"{RFT}." - f"{RDA}." - f"{int(shutters)}." - f"{SWR}." - f"null." - f"null." - f"{roof_cover}." - f"{int(BIM['TerrainRoughness'])}" - ) + bldg_config = f"M.SF." \ + f"{int(stories)}." \ + f"{BIM['RoofShape']}." \ + f"{RWC}." \ + f"{RFT}." \ + f"{RDA}." \ + f"{int(shutters)}." \ + f"{SWR}." \ + f"null." \ + f"null." \ + f"{roof_cover}." \ + f"{int(BIM['TerrainRoughness'])}" return bldg_config diff --git a/pelicun/tests/dl_calculation/rulesets/WindMetaVarRulesets.py b/pelicun/tests/dl_calculation/rulesets/WindMetaVarRulesets.py index 0ea56b739..baf5108d8 100644 --- a/pelicun/tests/dl_calculation/rulesets/WindMetaVarRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/WindMetaVarRulesets.py @@ -43,9 +43,11 @@ # Meredith Lockhead # Tracy Kijewski-Correa +import random import numpy as np import pandas as pd - +import datetime +import math def parse_BIM(BIM_in, location, hazards): """ @@ -99,46 +101,56 @@ def parse_BIM(BIM_in, location, hazards): # maps roof type to the internal representation ap_RoofType = { - 'hip': 'hip', + 'hip' : 'hip', 'hipped': 'hip', - 'Hip': 'hip', + 'Hip' : 'hip', 'gabled': 'gab', - 'gable': 'gab', - 'Gable': 'gab', - 'flat': 'flt', - 'Flat': 'flt', + 'gable' : 'gab', + 'Gable' : 'gab', + 'flat' : 'flt', + 'Flat' : 'flt' } # maps roof system to the internal representation - ap_RoofSyste = {'Wood': 'trs', 'OWSJ': 'ows', 'N/A': 'trs'} - roof_system = BIM_in.get('RoofSystem', 'Wood') + ap_RoofSyste = { + 'Wood': 'trs', + 'OWSJ': 'ows', + 'N/A': 'trs' + } + roof_system = BIM_in.get('RoofSystem','Wood') if pd.isna(roof_system): roof_system = 'Wood' - # flake8 - unused variable: `ap_NoUnits`. - # # maps number of units to the internal representation - # ap_NoUnits = { - # 'Single': 'sgl', - # 'Multiple': 'mlt', - # 'Multi': 'mlt', - # 'nav': 'nav', - # } - - # maps for design level (Marginal Engineered is mapped to - # Engineered as default) - ap_DesignLevel = {'E': 'E', 'NE': 'NE', 'PE': 'PE', 'ME': 'E'} - design_level = BIM_in.get('DesignLevel', 'E') + # maps number of units to the internal representation + ap_NoUnits = { + 'Single': 'sgl', + 'Multiple': 'mlt', + 'Multi': 'mlt', + 'nav': 'nav' + } + + # maps for design level (Marginal Engineered is mapped to Engineered as default) + ap_DesignLevel = { + 'E': 'E', + 'NE': 'NE', + 'PE': 'PE', + 'ME': 'E' + } + design_level = BIM_in.get('DesignLevel','E') if pd.isna(design_level): design_level = 'E' # Average January Temp. - ap_ajt = {'Above': 'above', 'Below': 'below'} + ap_ajt = { + 'Above': 'above', + 'Below': 'below' + } # Year built alname_yearbuilt = ['YearBuiltNJDEP', 'yearBuilt', 'YearBuiltMODIV'] yearbuilt = None try: yearbuilt = BIM_in['YearBuilt'] - except KeyError: + except: for i in alname_yearbuilt: if i in BIM_in.keys(): yearbuilt = BIM_in[i] @@ -149,12 +161,7 @@ def parse_BIM(BIM_in, location, hazards): yearbuilt = 1985 # Number of Stories - alname_nstories = [ - 'stories', - 'NumberofStories0', - 'NumberofStories', - 'NumberofStories1', - ] + alname_nstories = ['stories', 'NumberofStories0', 'NumberofStories', 'NumberofStories1'] nstories = None try: nstories = BIM_in['NumberOfStories'] @@ -193,6 +200,7 @@ def parse_BIM(BIM_in, location, hazards): dws = BIM_in[alname] break + alname_occupancy = ['occupancy'] oc = None try: @@ -230,9 +238,9 @@ def parse_BIM(BIM_in, location, hazards): 6113: 'OW', 6114: 'D', 6115: 'NA', - 6119: 'NA', + 6119: 'NA' } - if isinstance(BIM_in['FloodZone'], int): + if type(BIM_in['FloodZone']) == int: # NJDEP code for flood zone (conversion to the FEMA designations) floodzone_fema = ap_FloodZone[BIM_in['FloodZone']] else: @@ -257,50 +265,44 @@ def parse_BIM(BIM_in, location, hazards): buildingtype = BIM_in['BuildingType'] # first, pull in the provided data - BIM_ap.update( - dict( - OccupancyClass=str(oc), - BuildingType=buildingtype, - YearBuilt=int(yearbuilt), - # double check with Tracy for format - (NumberStories0 - # is 4-digit code) - # (NumberStories1 is image-processed story number) - NumberOfStories=int(nstories), - PlanArea=float(area), - FloodZone=floodzone_fema, - V_ult=float(dws), - AvgJanTemp=ap_ajt[BIM_in.get('AvgJanTemp', 'Below')], - RoofShape=ap_RoofType[BIM_in['RoofShape']], - RoofSlope=float(BIM_in.get('RoofSlope', 0.25)), # default 0.25 - SheathingThickness=float( - BIM_in.get('SheathingThick', 1.0) - ), # default 1.0 - RoofSystem=str( - ap_RoofSyste[roof_system] - ), # only valid for masonry structures - Garage=float(BIM_in.get('Garage', -1.0)), - LULC=BIM_in.get('LULC', -1), - z0=float( - BIM_in.get('z0', -1) - ), # if the z0 is already in the input file - Terrain=BIM_in.get('Terrain', -1), - MeanRoofHt=float(BIM_in.get('MeanRoofHt', 15.0)), # default 15 - DesignLevel=str(ap_DesignLevel[design_level]), # default engineered - WindowArea=float(BIM_in.get('WindowArea', 0.20)), - WoodZone=str(BIM_in.get('WindZone', 'I')), - ) - ) + BIM_ap.update(dict( + OccupancyClass=str(oc), + BuildingType=buildingtype, + YearBuilt=int(yearbuilt), + # double check with Tracy for format - (NumberStories0 is 4-digit code) + # (NumberStories1 is image-processed story number) + NumberOfStories=int(nstories), + PlanArea=float(area), + FloodZone=floodzone_fema, + V_ult=float(dws), + AvgJanTemp=ap_ajt[BIM_in.get('AvgJanTemp','Below')], + RoofShape=ap_RoofType[BIM_in['RoofShape']], + RoofSlope=float(BIM_in.get('RoofSlope',0.25)), # default 0.25 + SheathingThickness=float(BIM_in.get('SheathingThick',1.0)), # default 1.0 + RoofSystem=str(ap_RoofSyste[roof_system]), # only valid for masonry structures + Garage=float(BIM_in.get('Garage',-1.0)), + LULC=BIM_in.get('LULC',-1), + z0 = float(BIM_in.get('z0',-1)), # if the z0 is already in the input file + Terrain = BIM_in.get('Terrain',-1), + MeanRoofHt=float(BIM_in.get('MeanRoofHt',15.0)), # default 15 + DesignLevel=str(ap_DesignLevel[design_level]), # default engineered + WindowArea=float(BIM_in.get('WindowArea',0.20)), + WoodZone=str(BIM_in.get('WindZone', 'I')) + )) if 'inundation' in hazards: # maps for split level - ap_SplitLevel = {'NO': 0, 'YES': 1} + ap_SplitLevel = { + 'NO': 0, + 'YES': 1 + } - foundation = BIM_in.get('FoundationType', 3501) + foundation = BIM_in.get('FoundationType',3501) if pd.isna(foundation): foundation = 3501 - nunits = BIM_in.get('NoUnits', 1) + nunits = BIM_in.get('NoUnits',1) if pd.isna(nunits): nunits = 1 @@ -323,9 +325,9 @@ def parse_BIM(BIM_in, location, hazards): 6113: 'OW', 6114: 'D', 6115: 'NA', - 6119: 'NA', + 6119: 'NA' } - if isinstance(BIM_in['FloodZone'], int): + if type(BIM_in['FloodZone']) == int: # NJDEP code for flood zone (conversion to the FEMA designations) floodzone_fema = ap_FloodZone[BIM_in['FloodZone']] else: @@ -333,19 +335,15 @@ def parse_BIM(BIM_in, location, hazards): floodzone_fema = BIM_in['FloodZone'] # add the parsed data to the BIM dict - BIM_ap.update( - dict( - DesignLevel=str(ap_DesignLevel[design_level]), # default engineered - NumberOfUnits=int(nunits), - FirstFloorElevation=float(BIM_in.get('FirstFloorHt1', 10.0)), - SplitLevel=bool( - ap_SplitLevel[BIM_in.get('SplitLevel', 'NO')] - ), # dfault: no - FoundationType=int(foundation), # default: pile - City=BIM_in.get('City', 'NA'), - FloodZone=str(floodzone_fema), - ) - ) + BIM_ap.update(dict( + DesignLevel=str(ap_DesignLevel[design_level]), # default engineered + NumberOfUnits=int(nunits), + FirstFloorElevation=float(BIM_in.get('FirstFloorHt1',10.0)), + SplitLevel=bool(ap_SplitLevel[BIM_in.get('SplitLevel','NO')]), # dfault: no + FoundationType=int(foundation), # default: pile + City=BIM_in.get('City','NA'), + FloodZone =str(floodzone_fema) + )) # add inferred, generic meta-variables @@ -372,12 +370,12 @@ def parse_BIM(BIM_in, location, hazards): # The flood_lim and general_lim limits depend on the year of construction if BIM_ap['YearBuilt'] >= 2016: # In IRC 2015: - flood_lim = 130.0 # mph - general_lim = 140.0 # mph + flood_lim = 130.0 # mph + general_lim = 140.0 # mph else: # In IRC 2009 and earlier versions - flood_lim = 110.0 # mph - general_lim = 120.0 # mph + flood_lim = 110.0 # mph + general_lim = 120.0 # mph # Areas within hurricane-prone regions located in accordance with # one of the following: # (1) Within 1 mile (1.61 km) of the coastal mean high water line @@ -387,13 +385,8 @@ def parse_BIM(BIM_in, location, hazards): if not HPR: WBD = False else: - WBD = ( - ( - BIM_ap['FloodZone'].startswith('A') - or BIM_ap['FloodZone'].startswith('V') - ) - and BIM_ap['V_ult'] >= flood_lim - ) or (BIM_ap['V_ult'] >= general_lim) + WBD = (((BIM_ap['FloodZone'].startswith('A') or BIM_ap['FloodZone'].startswith('V')) and + BIM_ap['V_ult'] >= flood_lim) or (BIM_ap['V_ult'] >= general_lim)) # Terrain # open (0.03) = 3 @@ -405,86 +398,68 @@ def parse_BIM(BIM_in, location, hazards): # digidownload/metadata/lulc02/anderson2002.html) by T. Wu group # (see internal report on roughness calculations, Table 4). # These are mapped to Hazus defintions as follows: - # Open Water (5400s) with zo=0.01 and barren land (7600) with - # zo=0.04 assume Open Open Space Developed, Low Intensity - # Developed, Medium Intensity Developed (1110-1140) assumed - # zo=0.35-0.4 assume Suburban High Intensity Developed (1600) - # with zo=0.6 assume Lt. Tree Forests of all classes - # (4100-4300) assumed zo=0.6 assume Lt. Tree Shrub (4400) with - # zo=0.06 assume Open Grasslands, pastures and agricultural - # areas (2000 series) with zo=0.1-0.15 assume Lt. Suburban - # Woody Wetlands (6250) with zo=0.3 assume suburban Emergent - # Herbaceous Wetlands (6240) with zo=0.03 assume Open + # Open Water (5400s) with zo=0.01 and barren land (7600) with zo=0.04 assume Open + # Open Space Developed, Low Intensity Developed, Medium Intensity Developed + # (1110-1140) assumed zo=0.35-0.4 assume Suburban + # High Intensity Developed (1600) with zo=0.6 assume Lt. Tree + # Forests of all classes (4100-4300) assumed zo=0.6 assume Lt. Tree + # Shrub (4400) with zo=0.06 assume Open + # Grasslands, pastures and agricultural areas (2000 series) with + # zo=0.1-0.15 assume Lt. Suburban + # Woody Wetlands (6250) with zo=0.3 assume suburban + # Emergent Herbaceous Wetlands (6240) with zo=0.03 assume Open # Note: HAZUS category of trees (1.00) does not apply to any LU/LC in NJ - terrain = 15 # Default in Reorganized Rulesets - WIND + terrain = 15 # Default in Reorganized Rulesets - WIND LULC = BIM_ap['LULC'] TER = BIM_ap['Terrain'] - if BIM_ap['z0'] > 0: + if (BIM_ap['z0'] > 0): terrain = int(100 * BIM_ap['z0']) - elif LULC > 0: - if BIM_ap['FloodZone'].startswith('V') or BIM_ap['FloodZone'] in [ - 'A', - 'AE', - 'A1-30', - 'AR', - 'A99', - ]: + elif (LULC > 0): + if (BIM_ap['FloodZone'].startswith('V') or BIM_ap['FloodZone'] in ['A', 'AE', 'A1-30', 'AR', 'A99']): terrain = 3 - elif (LULC >= 5000) and (LULC <= 5999): - terrain = 3 # Open + elif ((LULC >= 5000) and (LULC <= 5999)): + terrain = 3 # Open elif ((LULC == 4400) or (LULC == 6240)) or (LULC == 7600): - terrain = 3 # Open - elif (LULC >= 2000) and (LULC <= 2999): - terrain = 15 # Light suburban - elif ((LULC >= 1110) and (LULC <= 1140)) or ( - (LULC >= 6250) and (LULC <= 6252) - ): - terrain = 35 # Suburban + terrain = 3 # Open + elif ((LULC >= 2000) and (LULC <= 2999)): + terrain = 15 # Light suburban + elif ((LULC >= 1110) and (LULC <= 1140)) or ((LULC >= 6250) and (LULC <= 6252)): + terrain = 35 # Suburban elif ((LULC >= 4100) and (LULC <= 4300)) or (LULC == 1600): - terrain = 70 # light trees - elif TER > 0: - if BIM_ap['FloodZone'].startswith('V') or BIM_ap['FloodZone'] in [ - 'A', - 'AE', - 'A1-30', - 'AR', - 'A99', - ]: + terrain = 70 # light trees + elif (TER > 0): + if (BIM_ap['FloodZone'].startswith('V') or BIM_ap['FloodZone'] in ['A', 'AE', 'A1-30', 'AR', 'A99']): terrain = 3 - elif (TER >= 50) and (TER <= 59): - terrain = 3 # Open + elif ((TER >= 50) and (TER <= 59)): + terrain = 3 # Open elif ((TER == 44) or (TER == 62)) or (TER == 76): - terrain = 3 # Open - elif (TER >= 20) and (TER <= 29): - terrain = 15 # Light suburban + terrain = 3 # Open + elif ((TER >= 20) and (TER <= 29)): + terrain = 15 # Light suburban elif (TER == 11) or (TER == 61): - terrain = 35 # Suburban + terrain = 35 # Suburban elif ((TER >= 41) and (TER <= 43)) or (TER in [16, 17]): - terrain = 70 # light trees - - BIM_ap.update( - dict( - # Nominal Design Wind Speed - # Former term was “Basic Wind Speed”; it is now the “Nominal Design - # Wind Speed (V_asd). Unit: mph." - V_asd=np.sqrt(0.6 * BIM_ap['V_ult']), - HazardProneRegion=HPR, - WindBorneDebris=WBD, - TerrainRoughness=terrain, - ) - ) + terrain = 70 # light trees + + BIM_ap.update(dict( + # Nominal Design Wind Speed + # Former term was “Basic Wind Speed”; it is now the “Nominal Design + # Wind Speed (V_asd). Unit: mph." + V_asd = np.sqrt(0.6 * BIM_ap['V_ult']), + + HazardProneRegion=HPR, + WindBorneDebris=WBD, + TerrainRoughness=terrain, + )) if 'inundation' in hazards: - BIM_ap.update( - dict( - # Flood Risk - # Properties in the High Water Zone (within 1 mile of - # the coast) are at risk of flooding and other - # wind-borne debris action. - # TODO: need high water zone for this and move it to inputs! - FloodRisk=True, - ) - ) + BIM_ap.update(dict( + # Flood Risk + # Properties in the High Water Zone (within 1 mile of the coast) are at + # risk of flooding and other wind-borne debris action. + FloodRisk=True, # TODO: need high water zone for this and move it to inputs! + )) return BIM_ap + diff --git a/pelicun/tests/dl_calculation/rulesets/WindSECBRulesets.py b/pelicun/tests/dl_calculation/rulesets/WindSECBRulesets.py index db68ad03f..d07f63fdf 100644 --- a/pelicun/tests/dl_calculation/rulesets/WindSECBRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/WindSECBRulesets.py @@ -45,7 +45,6 @@ import random - def SECB_config(BIM): """ Rules to identify a HAZUS SECB configuration based on BIM data @@ -62,7 +61,7 @@ def SECB_config(BIM): class. """ - year = BIM['YearBuilt'] # just for the sake of brevity + year = BIM['YearBuilt'] # just for the sake of brevity # Roof cover if BIM['RoofShape'] in ['gab', 'hip']: @@ -95,13 +94,14 @@ def SECB_config(BIM): # Wind Debris (widd in HAZSU) # HAZUS A: Res/Comm, B: Varies by direction, C: Residential, D: None - WIDD = 'C' # residential (default) - if BIM['OccupancyClass'] in ['RES1', 'RES2', 'RES3A', 'RES3B', 'RES3C', 'RES3D']: - WIDD = 'C' # residential + WIDD = 'C' # residential (default) + if BIM['OccupancyClass'] in ['RES1', 'RES2', 'RES3A', 'RES3B', 'RES3C', + 'RES3D']: + WIDD = 'C' # residential elif BIM['OccupancyClass'] == 'AGR1': - WIDD = 'D' # None + WIDD = 'D' # None else: - WIDD = 'A' # Res/Comm + WIDD = 'A' # Res/Comm # Window area ratio if BIM['WindowArea'] < 0.33: @@ -131,24 +131,21 @@ def SECB_config(BIM): bldg_tag = 'S.ECB.H' # extend the BIM dictionary - BIM.update( - dict( - RoofCover=roof_cover, - WindowAreaRatio=WWR, - RoofDeckAttachmentM=MRDA, - Shutters=shutters, - WindDebrisClass=WIDD, - ) - ) - - bldg_config = ( - f"{bldg_tag}." - f"{roof_cover}." - f"{int(shutters)}." - f"{WIDD}." - f"{MRDA}." - f"{WWR}." - f"{int(BIM['TerrainRoughness'])}" - ) + BIM.update(dict( + RoofCover = roof_cover, + WindowAreaRatio = WWR, + RoofDeckAttachmentM = MRDA, + Shutters = shutters, + WindDebrisClass=WIDD + )) + bldg_config = f"{bldg_tag}." \ + f"{roof_cover}." \ + f"{int(shutters)}." \ + f"{WIDD}." \ + f"{MRDA}." \ + f"{WWR}." \ + f"{int(BIM['TerrainRoughness'])}" + return bldg_config + diff --git a/pelicun/tests/dl_calculation/rulesets/WindSERBRulesets.py b/pelicun/tests/dl_calculation/rulesets/WindSERBRulesets.py index 6c078dd15..d6711b347 100644 --- a/pelicun/tests/dl_calculation/rulesets/WindSERBRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/WindSERBRulesets.py @@ -45,7 +45,6 @@ import random - def SERB_config(BIM): """ Rules to identify a HAZUS SERB configuration based on BIM data @@ -62,7 +61,7 @@ def SERB_config(BIM): class. """ - year = BIM['YearBuilt'] # just for the sake of brevity + year = BIM['YearBuilt'] # just for the sake of brevity # Roof cover if BIM['RoofShape'] in ['gab', 'hip']: @@ -95,13 +94,14 @@ def SERB_config(BIM): # Wind Debris (widd in HAZSU) # HAZUS A: Res/Comm, B: Varies by direction, C: Residential, D: None - WIDD = 'C' # residential (default) - if BIM['OccupancyClass'] in ['RES1', 'RES2', 'RES3A', 'RES3B', 'RES3C', 'RES3D']: - WIDD = 'C' # residential + WIDD = 'C' # residential (default) + if BIM['OccupancyClass'] in ['RES1', 'RES2', 'RES3A', 'RES3B', 'RES3C', + 'RES3D']: + WIDD = 'C' # residential elif BIM['OccupancyClass'] == 'AGR1': - WIDD = 'D' # None + WIDD = 'D' # None else: - WIDD = 'A' # Res/Comm + WIDD = 'A' # Res/Comm # Window area ratio if BIM['WindowArea'] < 0.33: @@ -131,24 +131,20 @@ def SERB_config(BIM): bldg_tag = 'S.ERB.H' # extend the BIM dictionary - BIM.update( - dict( - RoofCover=roof_cover, - WindowAreaRatio=WWR, - RoofDeckAttachmentM=MRDA, - Shutters=shutters, - WindDebrisClass=WIDD, - ) - ) + BIM.update(dict( + RoofCover = roof_cover, + WindowAreaRatio = WWR, + RoofDeckAttachmentM = MRDA, + Shutters = shutters, + WindDebrisClass=WIDD + )) - bldg_config = ( - f"{bldg_tag}." - f"{roof_cover}." - f"{int(shutters)}." - f"{WIDD}." - f"{MRDA}." - f"{WWR}." - f"{int(BIM['TerrainRoughness'])}" - ) + bldg_config = f"{bldg_tag}." \ + f"{roof_cover}." \ + f"{int(shutters)}." \ + f"{WIDD}." \ + f"{MRDA}." \ + f"{WWR}." \ + f"{int(BIM['TerrainRoughness'])}" return bldg_config diff --git a/pelicun/tests/dl_calculation/rulesets/WindSPMBRulesets.py b/pelicun/tests/dl_calculation/rulesets/WindSPMBRulesets.py index 58b3e2b6a..42f8a6407 100644 --- a/pelicun/tests/dl_calculation/rulesets/WindSPMBRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/WindSPMBRulesets.py @@ -44,6 +44,7 @@ # Tracy Kijewski-Correa import random +import numpy as np import datetime @@ -63,7 +64,7 @@ def SPMB_config(BIM): class. """ - year = BIM['YearBuilt'] # just for the sake of brevity + year = BIM['YearBuilt'] # just for the sake of brevity # Roof Deck Age (~ Roof Quality) if BIM['YearBuilt'] >= (datetime.datetime.now().year - 50): @@ -109,16 +110,17 @@ def SPMB_config(BIM): bldg_tag = 'S.PMB.L' # extend the BIM dictionary - BIM.update( - dict(RoofQuality=roof_quality, RoofDeckAttachmentM=MRDA, Shutters=shutters) - ) + BIM.update(dict( + RoofQuality = roof_quality, + RoofDeckAttachmentM = MRDA, + Shutters = shutters + )) - bldg_config = ( - f"{bldg_tag}." - f"{int(shutters)}." - f"{roof_quality}." - f"{MRDA}." - f"{int(BIM['TerrainRoughness'])}" - ) + bldg_config = f"{bldg_tag}." \ + f"{int(shutters)}." \ + f"{roof_quality}." \ + f"{MRDA}." \ + f"{int(BIM['TerrainRoughness'])}" return bldg_config + diff --git a/pelicun/tests/dl_calculation/rulesets/WindWMUHRulesets.py b/pelicun/tests/dl_calculation/rulesets/WindWMUHRulesets.py index 23c62c341..6d5fe338d 100644 --- a/pelicun/tests/dl_calculation/rulesets/WindWMUHRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/WindWMUHRulesets.py @@ -46,7 +46,6 @@ import random import datetime - def WMUH_config(BIM): """ Rules to identify a HAZUS WMUH configuration based on BIM data @@ -66,15 +65,15 @@ def WMUH_config(BIM): year = BIM['YearBuilt'] # just for the sake of brevity # Secondary Water Resistance (SWR) - SWR = 0 # Default + SWR = 0 # Default if year > 2000: if BIM['RoofShape'] == 'flt': - SWR = 'null' # because SWR is not a question for flat roofs - elif BIM['RoofShape'] in ['gab', 'hip']: + SWR = 'null' # because SWR is not a question for flat roofs + elif BIM['RoofShape'] in ['gab','hip']: SWR = int(random.random() < 0.6) elif year > 1987: if BIM['RoofShape'] == 'flt': - SWR = 'null' # because SWR is not a question for flat roofs + SWR = 'null' # because SWR is not a question for flat roofs elif (BIM['RoofShape'] == 'gab') or (BIM['RoofShape'] == 'hip'): if BIM['RoofSlope'] < 0.33: SWR = int(True) @@ -83,7 +82,7 @@ def WMUH_config(BIM): else: # year <= 1987 if BIM['RoofShape'] == 'flt': - SWR = 'null' # because SWR is not a question for flat roofs + SWR = 'null' # because SWR is not a question for flat roofs else: SWR = int(random.random() < 0.3) @@ -140,7 +139,7 @@ def WMUH_config(BIM): # The base rule was then extended to the exposures closest to suburban and # light suburban, even though these are not considered by the code. if year > 2009: - if BIM['TerrainRoughness'] >= 35: # suburban or light trees + if BIM['TerrainRoughness'] >= 35: # suburban or light trees if BIM['V_ult'] > 168.0: RDA = '8s' # 8d @ 6"/6" 'D' else: @@ -166,7 +165,7 @@ def WMUH_config(BIM): # Attachment requirements are given based on sheathing thickness, basic # wind speed, and the mean roof height of the building. elif year > 1996: - if (BIM['V_ult'] >= 103) and (BIM['MeanRoofHt'] >= 25.0): + if (BIM['V_ult'] >= 103 ) and (BIM['MeanRoofHt'] >= 25.0): RDA = '8s' # 8d @ 6"/6" 'D' else: RDA = '8d' # 8d @ 6"/12" 'B' @@ -183,9 +182,9 @@ def WMUH_config(BIM): else: # year <= 1993 if BIM['SheathingThickness'] <= 0.5: - RDA = '6d' # 6d @ 6"/12" 'A' + RDA = '6d' # 6d @ 6"/12" 'A' else: - RDA = '8d' # 8d @ 6"/12" 'B' + RDA = '8d' # 8d @ 6"/12" 'B' # Roof-Wall Connection (RWC) # IRC 2000-2015: @@ -250,28 +249,25 @@ def WMUH_config(BIM): stories = min(BIM['NumberOfStories'], 3) # extend the BIM dictionary - BIM.update( - dict( - SecondaryWaterResistance=SWR, - RoofCover=roof_cover, - RoofQuality=roof_quality, - RoofDeckAttachmentW=RDA, - RoofToWallConnection=RWC, - Shutters=shutters, - ) - ) + BIM.update(dict( + SecondaryWaterResistance = SWR, + RoofCover = roof_cover, + RoofQuality = roof_quality, + RoofDeckAttachmentW = RDA, + RoofToWallConnection = RWC, + Shutters = shutters + )) - bldg_config = ( - f"W.MUH." - f"{int(stories)}." - f"{BIM['RoofShape']}." - f"{roof_cover}." - f"{roof_quality}." - f"{SWR}." - f"{RDA}." - f"{RWC}." - f"{int(shutters)}." - f"{int(BIM['TerrainRoughness'])}" - ) + bldg_config = f"W.MUH." \ + f"{int(stories)}." \ + f"{BIM['RoofShape']}." \ + f"{roof_cover}." \ + f"{roof_quality}." \ + f"{SWR}." \ + f"{RDA}." \ + f"{RWC}." \ + f"{int(shutters)}." \ + f"{int(BIM['TerrainRoughness'])}" return bldg_config + diff --git a/pelicun/tests/dl_calculation/rulesets/WindWSFRulesets.py b/pelicun/tests/dl_calculation/rulesets/WindWSFRulesets.py index f0dfbab14..957ecbf9c 100644 --- a/pelicun/tests/dl_calculation/rulesets/WindWSFRulesets.py +++ b/pelicun/tests/dl_calculation/rulesets/WindWSFRulesets.py @@ -46,7 +46,6 @@ import random import datetime - def WSF_config(BIM): """ Rules to identify a HAZUS WSF configuration based on BIM data @@ -63,12 +62,12 @@ def WSF_config(BIM): class. """ - year = BIM['YearBuilt'] # just for the sake of brevity + year = BIM['YearBuilt'] # just for the sake of brevity # Secondary Water Resistance (SWR) # Minimum drainage recommendations are in place in NJ (See below). # However, SWR indicates a code-plus practice. - SWR = False # Default in Reorganzied Rulesets - WIND + SWR = False # Default in Reorganzied Rulesets - WIND if year > 2000: # For buildings built after 2000, SWR is based on homeowner compliance # data from NC Coastal Homeowner Survey (2017) to capture potential @@ -92,13 +91,13 @@ def WSF_config(BIM): # Almost all other roof types require underlayment of some sort, but # the ruleset is based on asphalt shingles because it is most # conservative. - if BIM['RoofShape'] == 'flt': # note there is actually no 'flt' + if BIM['RoofShape'] == 'flt': # note there is actually no 'flt' SWR = True - elif BIM['RoofShape'] in ['gab', 'hip']: + elif BIM['RoofShape'] in ['gab','hip']: if BIM['RoofSlope'] <= 0.17: SWR = True elif BIM['RoofSlope'] < 0.33: - SWR = BIM['AvgJanTemp'] == 'below' + SWR = (BIM['AvgJanTemp'] == 'below') # Roof Deck Attachment (RDA) # IRC codes: @@ -108,46 +107,34 @@ def WSF_config(BIM): # codes. Commentary for Table R602.3(1) indicates 8d nails with 6”/6” # spacing (enhanced roof spacing) for ultimate wind speeds greater than # a speed_lim. speed_lim depends on the year of construction - RDA = '6d' # Default (aka A) in Reorganized Rulesets - WIND + RDA = '6d' # Default (aka A) in Reorganized Rulesets - WIND if year > 2000: if year >= 2016: # IRC 2015 - speed_lim = 130.0 # mph + speed_lim = 130.0 # mph else: # IRC 2000 - 2009 - speed_lim = 100.0 # mph + speed_lim = 100.0 # mph if BIM['V_ult'] > speed_lim: RDA = '8s' # 8d @ 6"/6" ('D' in the Reorganized Rulesets - WIND) else: RDA = '8d' # 8d @ 6"/12" ('B' in the Reorganized Rulesets - WIND) elif year > 1995: - if (BIM['SheathingThickness'] >= 0.3125) and ( - BIM['SheathingThickness'] <= 0.5 - ): - RDA = '6d' # 6d @ 6"/12" ('A' in the Reorganized Rulesets - WIND) - elif (BIM['SheathingThickness'] >= 0.59375) and ( - BIM['SheathingThickness'] <= 1.125 - ): - RDA = '8d' # 8d @ 6"/12" ('B' in the Reorganized Rulesets - WIND) + if ((BIM['SheathingThickness'] >= 0.3125) and (BIM['SheathingThickness'] <= 0.5)): + RDA = '6d' # 6d @ 6"/12" ('A' in the Reorganized Rulesets - WIND) + elif ((BIM['SheathingThickness'] >= 0.59375) and (BIM['SheathingThickness'] <= 1.125)): + RDA = '8d' # 8d @ 6"/12" ('B' in the Reorganized Rulesets - WIND) elif year > 1986: - if (BIM['SheathingThickness'] >= 0.3125) and ( - BIM['SheathingThickness'] <= 0.5 - ): - RDA = '6d' # 6d @ 6"/12" ('A' in the Reorganized Rulesets - WIND) - elif (BIM['SheathingThickness'] >= 0.59375) and ( - BIM['SheathingThickness'] <= 1.0 - ): - RDA = '8d' # 8d @ 6"/12" ('B' in the Reorganized Rulesets - WIND) + if ((BIM['SheathingThickness'] >= 0.3125) and (BIM['SheathingThickness'] <= 0.5)): + RDA = '6d' # 6d @ 6"/12" ('A' in the Reorganized Rulesets - WIND) + elif ((BIM['SheathingThickness'] >= 0.59375) and (BIM['SheathingThickness'] <= 1.0)): + RDA = '8d' # 8d @ 6"/12" ('B' in the Reorganized Rulesets - WIND) else: # year <= 1986 - if (BIM['SheathingThickness'] >= 0.3125) and ( - BIM['SheathingThickness'] <= 0.5 - ): - RDA = '6d' # 6d @ 6"/12" ('A' in the Reorganized Rulesets - WIND) - elif (BIM['SheathingThickness'] >= 0.625) and ( - BIM['SheathingThickness'] <= 1.0 - ): - RDA = '8d' # 8d @ 6"/12" ('B' in the Reorganized Rulesets - WIND) + if ((BIM['SheathingThickness'] >= 0.3125) and (BIM['SheathingThickness'] <= 0.5)): + RDA = '6d' # 6d @ 6"/12" ('A' in the Reorganized Rulesets - WIND) + elif ((BIM['SheathingThickness'] >= 0.625) and (BIM['SheathingThickness'] <= 1.0)): + RDA = '8d' # 8d @ 6"/12" ('B' in the Reorganized Rulesets - WIND) # Roof-Wall Connection (RWC) # IRC 2015 @@ -196,7 +183,7 @@ def WSF_config(BIM): # buildings are toe nails before 1992. else: # year <= 1992 - RWC = 'tnail' # Toe-nail + RWC = 'tnail' # Toe-nail # Shutters # IRC 2000-2015: @@ -244,57 +231,54 @@ def WSF_config(BIM): if BIM['Garage'] == -1: # no garage data, using the default "standard" garage = 'std' - shutters = 0 # HAZUS ties standard garage to w/o shutters + shutters = 0 # HAZUS ties standard garage to w/o shutters else: if year > 2000: if shutters: if BIM['Garage'] < 1: garage = 'no' else: - garage = 'sup' # SFBC 1994 - shutters = 1 # HAZUS ties SFBC 1994 to with shutters + garage = 'sup' # SFBC 1994 + shutters = 1 # HAZUS ties SFBC 1994 to with shutters else: if BIM['Garage'] < 1: - garage = 'no' # None + garage = 'no' # None else: - garage = 'std' # Standard - shutters = 0 # HAZUS ties standard garage to w/o shutters + garage = 'std' # Standard + shutters = 0 # HAZUS ties standard garage to w/o shutters elif year > (datetime.datetime.now().year - 30): if BIM['Garage'] < 1: - garage = 'no' # None + garage = 'no' # None else: - garage = 'std' # Standard - shutters = 0 # HAZUS ties standard garage to w/o shutters + garage = 'std' # Standard + shutters = 0 # HAZUS ties standard garage to w/o shutters else: # year <= current year - 30 if BIM['Garage'] < 1: - garage = 'no' # None + garage = 'no' # None else: - garage = 'wkd' # Weak - shutters = 0 # HAZUS ties weak garage to w/o shutters + garage = 'wkd' # Weak + shutters = 0 # HAZUS ties weak garage to w/o shutters # extend the BIM dictionary - BIM.update( - dict( - SecondaryWaterResistance=SWR, - RoofDeckAttachmentW=RDA, - RoofToWallConnection=RWC, - Shutters=shutters, - Garage=garage, - ) - ) + BIM.update(dict( + SecondaryWaterResistance = SWR, + RoofDeckAttachmentW = RDA, + RoofToWallConnection = RWC, + Shutters = shutters, + Garage = garage + )) # building configuration tag - bldg_config = ( - f"W.SF." - f"{int(min(BIM['NumberOfStories'],2))}." - f"{BIM['RoofShape']}." - f"{int(SWR)}." - f"{RDA}." - f"{RWC}." - f"{garage}." - f"{int(shutters)}." - f"{int(BIM['TerrainRoughness'])}" - ) - + bldg_config = f"W.SF." \ + f"{int(min(BIM['NumberOfStories'],2))}." \ + f"{BIM['RoofShape']}." \ + f"{int(SWR)}." \ + f"{RDA}." \ + f"{RWC}." \ + f"{garage}." \ + f"{int(shutters)}." \ + f"{int(BIM['TerrainRoughness'])}" + return bldg_config + diff --git a/pelicun/tests/dl_calculation/rulesets/__init__.py b/pelicun/tests/dl_calculation/rulesets/__init__.py new file mode 100644 index 000000000..72c332008 --- /dev/null +++ b/pelicun/tests/dl_calculation/rulesets/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/tests/maintenance/__init__.py b/pelicun/tests/maintenance/__init__.py new file mode 100644 index 000000000..72c332008 --- /dev/null +++ b/pelicun/tests/maintenance/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/tests/maintenance/search_in_functions.py b/pelicun/tests/maintenance/search_in_functions.py index 9dadc9e96..f989ebd7c 100644 --- a/pelicun/tests/maintenance/search_in_functions.py +++ b/pelicun/tests/maintenance/search_in_functions.py @@ -1,8 +1,42 @@ -""" -Code inspection methods/functions. -""" +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . + +"""Code inspection methods/functions.""" + +from __future__ import annotations # noqa: I001 +from pathlib import Path -from __future__ import annotations import ast @@ -18,18 +52,18 @@ def visit_FunctionDef( Parameters ---------- - node : ast.FunctionDef + node: ast.FunctionDef The AST node representing the function definition. - filename : str + filename: str The path to the Python file to be searched. - search_string : str + search_string: str The string to search for within the function bodies. - functions_with_string : list[str] + functions_with_string: list[str] The list to append function names that contain the search string. """ - with open(filename, 'r', encoding='utf-8') as f: + with Path(filename).open(encoding='utf-8') as f: contents = f.read() function_code = ast.get_source_segment(contents, node) @@ -46,9 +80,9 @@ def find_functions_with_string(filename: str, search_string: str) -> list[str]: Parameters ---------- - filename : str + filename: str The path to the Python file to be searched. - search_string : str + search_string: str The string to search for within the function bodies. Returns @@ -56,8 +90,9 @@ def find_functions_with_string(filename: str, search_string: str) -> list[str]: list[str] A list of function names that contain the search string in their bodies. + """ - with open(filename, 'r', encoding='utf-8') as file: + with Path(filename).open(encoding='utf-8') as file: contents = file.read() tree = ast.parse(contents, filename=filename) diff --git a/pelicun/tests/util.py b/pelicun/tests/util.py index a2c9fb4a6..3505c5321 100644 --- a/pelicun/tests/util.py +++ b/pelicun/tests/util.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -33,26 +32,16 @@ # # You should have received a copy of the BSD 3-Clause License along with # pelicun. If not, see . -# -# Contributors: -# Adam Zsarnóczay -# John Vouvakis Manousakis -""" -These are utility functions for the unit and integration tests. -""" +"""These are utility functions for the unit and integration tests.""" from __future__ import annotations -import pickle -import os -# pylint: disable=useless-suppression -# pylint: disable=unused-variable -# pylint: disable=pointless-statement -# pylint: disable=missing-return-doc,missing-return-type-doc +import pickle # noqa: S403 +from pathlib import Path -def export_pickle(filepath, obj, makedirs=True): +def export_pickle(filepath, obj, makedirs=True) -> None: # noqa: ANN001, FBT002 """ Auxiliary function to export a pickle object. @@ -69,20 +58,20 @@ def export_pickle(filepath, obj, makedirs=True): """ # extract the directory name - dirname = os.path.dirname(filepath) + dirname = Path(filepath).parent # if making directories is requested, if makedirs: # and the path does not exist - if not os.path.exists(dirname): + if not Path(dirname).exists(): # create the directory - os.makedirs(dirname) + Path(dirname).mkdir(parents=True) # open the file with the given filepath - with open(filepath, 'wb') as f: + with Path(filepath).open('wb') as f: # and store the object in the file pickle.dump(obj, f) -def import_pickle(filepath): +def import_pickle(filepath): # noqa: ANN001, ANN201 """ Auxiliary function to import a pickle object. @@ -97,6 +86,6 @@ def import_pickle(filepath): """ # open the file with the given filepath - with open(filepath, 'rb') as f: + with Path(filepath).open('rb') as f: # and retrieve the pickled object - return pickle.load(f) + return pickle.load(f) # noqa: S301 diff --git a/pelicun/tests/validation/__init__.py b/pelicun/tests/validation/__init__.py new file mode 100644 index 000000000..72c332008 --- /dev/null +++ b/pelicun/tests/validation/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/tests/validation/inactive/3d_interpolation.py b/pelicun/tests/validation/inactive/3d_interpolation.py index 3c365539e..09a627ac6 100644 --- a/pelicun/tests/validation/inactive/3d_interpolation.py +++ b/pelicun/tests/validation/inactive/3d_interpolation.py @@ -1,7 +1,41 @@ +# noqa: N999 +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . + """ With this code we verify that scipy's `RegularGridInterpolator` does what we expect. -Created: `Sat Jun 1 03:07:28 PM PDT 2024` """ @@ -9,36 +43,42 @@ import pandas as pd from scipy.interpolate import RegularGridInterpolator -# Define domains -num_samples = 100 -dom1 = np.linspace(0, 1, num_samples) -dom2 = np.linspace(0, 1, num_samples) -dom3 = np.linspace(0, 1, num_samples) - -# Define 3D array -vg1, vg2, vg3 = np.meshgrid(dom1, dom2, dom3) -values = vg1 + np.sqrt(vg2) + np.sin(vg3) - -# Define test inputs for interpolation. -x1 = np.random.rand(10) -x2 = np.random.rand(10) -x3 = np.random.rand(10) -test_values = np.column_stack((x1, x2, x3)) - -# Create the interpolation function -interp_func = RegularGridInterpolator((dom1, dom2, dom3), values) - -# Perform the interpolation -interpolated_value = interp_func(test_values) - -# Compare output with the exact value. -df = pd.DataFrame( - { - 'exact': x1 + np.sqrt(x2) + np.sin(x3), - 'interpolated': interpolated_value, - } -) -print(df) - -# Note: This does work with a 2D case, and it could scale to more than -# 3 dimensions. + +def main(): + # Define domains + num_samples = 100 + dom1 = np.linspace(0, 1, num_samples) + dom2 = np.linspace(0, 1, num_samples) + dom3 = np.linspace(0, 1, num_samples) + + # Define 3D array + vg1, vg2, vg3 = np.meshgrid(dom1, dom2, dom3) + values = vg1 + np.sqrt(vg2) + np.sin(vg3) + + # Define test inputs for interpolation. + x1 = np.random.rand(10) + x2 = np.random.rand(10) + x3 = np.random.rand(10) + test_values = np.column_stack((x1, x2, x3)) + + # Create the interpolation function + interp_func = RegularGridInterpolator((dom1, dom2, dom3), values) + + # Perform the interpolation + interpolated_value = interp_func(test_values) + + # Compare output with the exact value. + df = pd.DataFrame( + { + 'exact': x1 + np.sqrt(x2) + np.sin(x3), + 'interpolated': interpolated_value, + } + ) + print(df) + + # Note: This does work with a 2D case, and it could scale to more than + # 3 dimensions. + + +if __name__ == '__main__': + main() diff --git a/pelicun/tests/validation/inactive/__init__.py b/pelicun/tests/validation/inactive/__init__.py new file mode 100644 index 000000000..72c332008 --- /dev/null +++ b/pelicun/tests/validation/inactive/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/tests/validation/inactive/pandas_convert_speed.py b/pelicun/tests/validation/inactive/pandas_convert_speed.py index 8f153ccb1..82e25414b 100644 --- a/pelicun/tests/validation/inactive/pandas_convert_speed.py +++ b/pelicun/tests/validation/inactive/pandas_convert_speed.py @@ -1,8 +1,41 @@ -import pandas as pd -import numpy as np +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . + import time -# pylint: disable=pointless-statement +import numpy as np +import pandas as pd def benchmark(): diff --git a/pelicun/tests/validation/inactive/readme.md b/pelicun/tests/validation/inactive/readme.md index d3d8900ec..ceb464bca 100644 --- a/pelicun/tests/validation/inactive/readme.md +++ b/pelicun/tests/validation/inactive/readme.md @@ -1,3 +1,3 @@ This directory contains code that is not meant to be tested or -updated, but was used to verify outputs of vairous external libraries +updated, but was used to verify outputs of various external libraries we utilize and ensure they are in line with our expectations. diff --git a/pelicun/tests/validation/v0/__init__.py b/pelicun/tests/validation/v0/__init__.py new file mode 100644 index 000000000..72c332008 --- /dev/null +++ b/pelicun/tests/validation/v0/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/tests/validation/v0/data/CMP_marginals.csv b/pelicun/tests/validation/v0/data/CMP_marginals.csv new file mode 100755 index 000000000..c752f72e0 --- /dev/null +++ b/pelicun/tests/validation/v0/data/CMP_marginals.csv @@ -0,0 +1,2 @@ +,Units,Location,Direction,Theta_0,Blocks,Comment +cmp.A,ea,1,1,1,,Testing component A diff --git a/pelicun/tests/validation/v0/data/loss_functions.csv b/pelicun/tests/validation/v0/data/loss_functions.csv new file mode 100644 index 000000000..23c7b1939 --- /dev/null +++ b/pelicun/tests/validation/v0/data/loss_functions.csv @@ -0,0 +1,2 @@ +-,DV-Unit,Demand-Directional,Demand-Offset,Demand-Type,Demand-Unit,LossFunction-Theta_0,LossFunction-Theta_1,LossFunction-Family +cmp.A-Cost,loss_ratio,1,0,Peak Floor Acceleration,g,"0.00,1000.00|0.00,1000.00",, diff --git a/pelicun/tests/validation/0/readme.md b/pelicun/tests/validation/v0/readme.md similarity index 100% rename from pelicun/tests/validation/0/readme.md rename to pelicun/tests/validation/v0/readme.md diff --git a/pelicun/tests/validation/0/test_validation_0.py b/pelicun/tests/validation/v0/test_validation_0.py similarity index 86% rename from pelicun/tests/validation/0/test_validation_0.py rename to pelicun/tests/validation/v0/test_validation_0.py index c035104b4..c43ab719e 100644 --- a/pelicun/tests/validation/0/test_validation_0.py +++ b/pelicun/tests/validation/v0/test_validation_0.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -33,10 +32,6 @@ # # You should have received a copy of the BSD 3-Clause License along with # pelicun. If not, see . -# -# Contributors: -# Adam Zsarnóczay -# John Vouvakis Manousakis """ Validation test on loss functions. @@ -49,18 +44,18 @@ """ from __future__ import annotations + import numpy as np import pandas as pd -import pelicun -from pelicun import assessment +from pelicun import assessment, file_io -def test_validation_loss_function(): +def test_validation_loss_function() -> None: sample_size = 100000 # initialize a pelicun assessment - asmnt = assessment.Assessment({"PrintLog": False, "Seed": 42}) + asmnt = assessment.Assessment({'PrintLog': False, 'Seed': 42}) # # Demands @@ -82,7 +77,7 @@ def test_validation_loss_function(): asmnt.demand.load_model({'marginals': demands}) - asmnt.demand.generate_sample({"SampleSize": sample_size}) + asmnt.demand.generate_sample({'SampleSize': sample_size}) # # Asset @@ -91,7 +86,7 @@ def test_validation_loss_function(): asmnt.stories = 1 cmp_marginals = pd.read_csv( - 'pelicun/tests/validation/0/data/CMP_marginals.csv', index_col=0 + 'pelicun/tests/validation/v0/data/CMP_marginals.csv', index_col=0 ) cmp_marginals['Blocks'] = cmp_marginals['Blocks'] asmnt.asset.load_cmp_model({'marginals': cmp_marginals}) @@ -113,23 +108,25 @@ def test_validation_loss_function(): loss_map = pd.DataFrame(['cmp.A'], columns=['Repair'], index=['cmp.A']) asmnt.loss.add_loss_map(loss_map) - loss_functions = pelicun.file_io.load_data( - 'pelicun/tests/validation/0/data/loss_functions.csv', + loss_functions = file_io.load_data( + 'pelicun/tests/validation/v0/data/loss_functions.csv', reindex=False, unit_conversion_factors=asmnt.unit_conversion_factors, ) + assert isinstance(loss_functions, pd.DataFrame) asmnt.loss.load_model_parameters([loss_functions]) asmnt.loss.calculate() loss, _ = asmnt.loss.aggregate_losses(future=True) + assert isinstance(loss, pd.DataFrame) - loss_vals = loss['repair_cost'].values + loss_vals = loss['repair_cost'].to_numpy() # sample median should be close to 0.05 assert np.allclose(np.median(loss_vals), 0.05, atol=1e-2) # dispersion should be close to 0.9 assert np.allclose(np.log(loss_vals).std(), 0.90, atol=1e-2) - # # TODO also test save/load sample + # TODO(JVM): also test save/load sample # asmnt.loss.save_sample('/tmp/sample.csv') # asmnt.loss.load_sample('/tmp/sample.csv') diff --git a/pelicun/tests/validation/v1/__init__.py b/pelicun/tests/validation/v1/__init__.py new file mode 100644 index 000000000..72c332008 --- /dev/null +++ b/pelicun/tests/validation/v1/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/tests/validation/v1/data/CMP_marginals.csv b/pelicun/tests/validation/v1/data/CMP_marginals.csv new file mode 100755 index 000000000..c752f72e0 --- /dev/null +++ b/pelicun/tests/validation/v1/data/CMP_marginals.csv @@ -0,0 +1,2 @@ +,Units,Location,Direction,Theta_0,Blocks,Comment +cmp.A,ea,1,1,1,,Testing component A diff --git a/pelicun/tests/validation/v1/data/damage_db.csv b/pelicun/tests/validation/v1/data/damage_db.csv new file mode 100644 index 000000000..50319d1d5 --- /dev/null +++ b/pelicun/tests/validation/v1/data/damage_db.csv @@ -0,0 +1,2 @@ +ID,Demand-Directional,Demand-Offset,Demand-Type,Demand-Unit,LS1-Family,LS1-Theta_0,LS1-Theta_1,LS2-Family,LS2-Theta_0,LS2-Theta_1 +cmp.A,1,0,Story Drift Ratio,rad,lognormal,0.015,0.5,lognormal,0.02,0.5 diff --git a/pelicun/tests/validation/1/readme.md b/pelicun/tests/validation/v1/readme.md similarity index 90% rename from pelicun/tests/validation/1/readme.md rename to pelicun/tests/validation/v1/readme.md index 19d3a511e..396db6f71 100644 --- a/pelicun/tests/validation/1/readme.md +++ b/pelicun/tests/validation/v1/readme.md @@ -17,4 +17,4 @@ If $\mathrm{Y} \sim \textrm{LogNormal}(\delta, \beta)$, then $\mathrm{X} = \log ``` where $\Phi$ is the cumulative distribution function of the standard normal distribution, $\delta_{C1}$, $\delta_{C2}$, $\beta_{C1}$, $\beta_{C2}$ are the medians and dispersions of the fragility curve capacities, and $\delta_{D}$, $\beta_{D}$ is the median and dispersion of the EDP demand. -The equations inherently asume that the capacity RVs for the damage states are perfectly correlated, which is the case for sequential damage states. +The equations inherently assume that the capacity RVs for the damage states are perfectly correlated, which is the case for sequential damage states. diff --git a/pelicun/tests/validation/1/test_validation_1.py b/pelicun/tests/validation/v1/test_validation_1.py similarity index 85% rename from pelicun/tests/validation/1/test_validation_1.py rename to pelicun/tests/validation/v1/test_validation_1.py index d9e840792..0b94b8598 100644 --- a/pelicun/tests/validation/1/test_validation_1.py +++ b/pelicun/tests/validation/v1/test_validation_1.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -33,10 +32,6 @@ # # You should have received a copy of the BSD 3-Clause License along with # pelicun. If not, see . -# -# Contributors: -# Adam Zsarnóczay -# John Vouvakis Manousakis """ Validation test for the probability of each damage state of a @@ -45,19 +40,20 @@ """ from __future__ import annotations + import tempfile + import numpy as np import pandas as pd -import pelicun -from pelicun import assessment from scipy.stats import norm # type: ignore +from pelicun import assessment, file_io -def test_validation_ds_probabilities(): +def test_validation_ds_probabilities() -> None: sample_size = 1000000 - asmnt = assessment.Assessment({"PrintLog": False, "Seed": 42}) + asmnt = assessment.Assessment({'PrintLog': False, 'Seed': 42}) # # Demands @@ -81,7 +77,7 @@ def test_validation_ds_probabilities(): asmnt.demand.load_model({'marginals': demands}) # generate samples - asmnt.demand.generate_sample({"SampleSize": sample_size}) + asmnt.demand.generate_sample({'SampleSize': sample_size}) # # Asset @@ -92,7 +88,7 @@ def test_validation_ds_probabilities(): # load component definitions cmp_marginals = pd.read_csv( - 'pelicun/tests/validation/1/data/CMP_marginals.csv', index_col=0 + 'pelicun/tests/validation/v1/data/CMP_marginals.csv', index_col=0 ) cmp_marginals['Blocks'] = cmp_marginals['Blocks'] asmnt.asset.load_cmp_model({'marginals': cmp_marginals}) @@ -104,13 +100,14 @@ def test_validation_ds_probabilities(): # Damage # - damage_db = pelicun.file_io.load_data( - 'pelicun/tests/validation/1/data/damage_db.csv', + damage_db = file_io.load_data( + 'pelicun/tests/validation/v1/data/damage_db.csv', reindex=False, unit_conversion_factors=asmnt.unit_conversion_factors, ) + assert isinstance(damage_db, pd.DataFrame) - cmp_set = asmnt.asset.list_unique_component_ids(as_set=True) + cmp_set = set(asmnt.asset.list_unique_component_ids()) # load the models into pelicun asmnt.damage.load_model_parameters([damage_db], cmp_set) @@ -150,14 +147,15 @@ def test_validation_ds_probabilities(): (demand_mean - capacity_2_mean) / np.sqrt(demand_std**2 + capacity_std**2) ) - assert np.allclose((probs[0]).values, p0, atol=1e-2) - assert np.allclose((probs[1]).values, p1, atol=1e-2) - assert np.allclose((probs[2]).values, p2, atol=1e-2) + assert np.allclose(probs.iloc[0, 0], p0, atol=1e-2) # type: ignore + assert np.allclose(probs.iloc[0, 1], p1, atol=1e-2) # type: ignore + assert np.allclose(probs.iloc[0, 2], p2, atol=1e-2) # type: ignore # # Also test load/save sample # + assert asmnt.damage.ds_model.sample is not None asmnt.damage.ds_model.sample = asmnt.damage.ds_model.sample.iloc[0:100, :] # (we reduce the number of realizations to conserve resources) before = asmnt.damage.ds_model.sample.copy() diff --git a/pelicun/tests/validation/v2/__init__.py b/pelicun/tests/validation/v2/__init__.py new file mode 100644 index 000000000..72c332008 --- /dev/null +++ b/pelicun/tests/validation/v2/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/tests/validation/v2/data/CMP_marginals.csv b/pelicun/tests/validation/v2/data/CMP_marginals.csv new file mode 100755 index 000000000..e085a1f00 --- /dev/null +++ b/pelicun/tests/validation/v2/data/CMP_marginals.csv @@ -0,0 +1,8 @@ +,Units,Location,Direction,Theta_0,Blocks,Comment +D.30.31.013i,ea,roof,0,1,,Chiller (parameters have been modified for testing) +missing.component,ea,0,1,1,,Testing +testing.component,ea,1,1,1,,Testing +testing.component.2,ea,1,1,2,4,Testing +collapse,ea,0,1,1,,Collapsed building +excessiveRID,ea,all,"1,2",1,,Excessive residual drift +irreparable,ea,0,1,1,,Irreparable building diff --git a/pelicun/tests/validation/v2/data/additional_consequences.csv b/pelicun/tests/validation/v2/data/additional_consequences.csv new file mode 100644 index 000000000..889ca6ae9 --- /dev/null +++ b/pelicun/tests/validation/v2/data/additional_consequences.csv @@ -0,0 +1,3 @@ +-,Incomplete-,Quantity-Unit,DV-Unit,DS1-Theta_0 +replacement-Cost,0,1 EA,USD_2011,21600000 +replacement-Time,0,1 EA,worker_day,12500 diff --git a/pelicun/tests/validation/v2/data/additional_damage_db.csv b/pelicun/tests/validation/v2/data/additional_damage_db.csv new file mode 100644 index 000000000..5d7c885ac --- /dev/null +++ b/pelicun/tests/validation/v2/data/additional_damage_db.csv @@ -0,0 +1,5 @@ +ID,Demand-Directional,Demand-Offset,Demand-Type,Demand-Unit,Incomplete-,LS1-DamageStateWeights,LS1-Family,LS1-Theta_0,LS1-Theta_1,LS2-DamageStateWeights,LS2-Family,LS2-Theta_0,LS2-Theta_1,LS3-DamageStateWeights,LS3-Family,LS3-Theta_0,LS3-Theta_1,LS4-DamageStateWeights,LS4-Family,LS4-Theta_0,LS4-Theta_1 +D.30.31.013i,0.0,0.0,Peak Floor Acceleration,g,0,0.340000 | 0.330000 | 0.330000,lognormal,0.40,0.5,,,,,,,,,,,, +excessiveRID,1.0,0.0,Residual Interstory Drift Ratio,rad,0,,lognormal,0.01,0.3,,,,,,,,,,,, +irreparable,1.0,0.0,Peak Spectral Acceleration|1.13,g,0,,,10000000000.0,,,,,,,,,,,,, +collapse,1.0,0.0,Peak Spectral Acceleration|1.13,g,0,,lognormal,1.50,0.5,,,,,,,,,,,, diff --git a/pelicun/tests/validation/v2/data/additional_loss_functions.csv b/pelicun/tests/validation/v2/data/additional_loss_functions.csv new file mode 100644 index 000000000..7d9bf4b93 --- /dev/null +++ b/pelicun/tests/validation/v2/data/additional_loss_functions.csv @@ -0,0 +1,5 @@ +-,DV-Unit,Demand-Directional,Demand-Offset,Demand-Type,Demand-Unit,LossFunction-Theta_0,LossFunction-Theta_1,LossFunction-Family +testing.component-Cost,USD_2011,0,0,Peak Floor Acceleration,g,"0.00,100000.00|0.00,5.00",0.3,lognormal +testing.component-Time,USD_2011,0,0,Peak Floor Acceleration,g,"0.00,50.00|0.00,5.00",0.3,lognormal +testing.component.2-Cost,USD_2011,0,0,Peak Floor Acceleration,g,"0.00,10.00|0.00,5.00",, +testing.component.2-Time,USD_2011,0,0,Peak Floor Acceleration,g,"0.00,5.00|0.00,5.00",, diff --git a/pelicun/tests/validation/v2/data/demand_data.csv b/pelicun/tests/validation/v2/data/demand_data.csv new file mode 100644 index 000000000..7f2efa3f0 --- /dev/null +++ b/pelicun/tests/validation/v2/data/demand_data.csv @@ -0,0 +1,19 @@ +--,Units,Family,Theta_0,Theta_1 +PFA-0-1,g,lognormal,0.45,0.40 +PFA-0-2,g,lognormal,0.45,0.40 +PFA-1-1,g,lognormal,0.45,0.40 +PFA-1-2,g,lognormal,0.45,0.40 +PFA-2-1,g,lognormal,0.45,0.40 +PFA-2-2,g,lognormal,0.45,0.40 +PFA-3-1,g,lognormal,0.45,0.40 +PFA-3-2,g,lognormal,0.45,0.40 +PFA-4-1,g,lognormal,0.45,0.40 +PFA-4-2,g,lognormal,0.45,0.40 +PID-1-1,rad,lognormal,0.03,0.35 +PID-1-2,rad,lognormal,0.03,0.35 +PID-2-1,rad,lognormal,0.03,0.35 +PID-2-2,rad,lognormal,0.03,0.35 +PID-3-1,rad,lognormal,0.03,0.35 +PID-3-2,rad,lognormal,0.03,0.35 +PID-4-1,rad,lognormal,0.03,0.35 +PID-4-2,rad,lognormal,0.03,0.35 diff --git a/pelicun/tests/validation/v2/data/loss_functions.csv b/pelicun/tests/validation/v2/data/loss_functions.csv new file mode 100644 index 000000000..23c7b1939 --- /dev/null +++ b/pelicun/tests/validation/v2/data/loss_functions.csv @@ -0,0 +1,2 @@ +-,DV-Unit,Demand-Directional,Demand-Offset,Demand-Type,Demand-Unit,LossFunction-Theta_0,LossFunction-Theta_1,LossFunction-Family +cmp.A-Cost,loss_ratio,1,0,Peak Floor Acceleration,g,"0.00,1000.00|0.00,1000.00",, diff --git a/pelicun/tests/validation/2/readme.md b/pelicun/tests/validation/v2/readme.md similarity index 100% rename from pelicun/tests/validation/2/readme.md rename to pelicun/tests/validation/v2/readme.md diff --git a/pelicun/tests/validation/2/test_validation_2.py b/pelicun/tests/validation/v2/test_validation_2.py similarity index 50% rename from pelicun/tests/validation/2/test_validation_2.py rename to pelicun/tests/validation/v2/test_validation_2.py index e2039b467..111113f7d 100644 --- a/pelicun/tests/validation/2/test_validation_2.py +++ b/pelicun/tests/validation/v2/test_validation_2.py @@ -1,3 +1,37 @@ +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . + """ Tests a complete loss estimation workflow combining damage state and loss function driven components. @@ -6,24 +40,24 @@ """ import tempfile + import numpy as np import pandas as pd import pytest -import pelicun -from pelicun.warnings import PelicunWarning -from pelicun import file_io -from pelicun import assessment +import pelicun +from pelicun import assessment, file_io +from pelicun.pelicun_warnings import PelicunWarning -def test_combined_workflow(): +def test_combined_workflow() -> None: temp_dir = tempfile.mkdtemp() sample_size = 10000 # Initialize a pelicun assessment asmnt = assessment.Assessment( - {"PrintLog": True, "Seed": 415, "LogFile": f'{temp_dir}/log_file.txt'} + {'PrintLog': True, 'Seed': 415, 'LogFile': f'{temp_dir}/log_file.txt'} ) asmnt.options.list_all_ds = True @@ -31,13 +65,15 @@ def test_combined_workflow(): asmnt.options.eco_scale['AcrossDamageStates'] = True demand_data = file_io.load_data( - 'pelicun/tests/validation/2/data/demand_data.csv', + 'pelicun/tests/validation/v2/data/demand_data.csv', unit_conversion_factors=None, reindex=False, ) ndims = len(demand_data) perfect_correlation = pd.DataFrame( - np.ones((ndims, ndims)), columns=demand_data.index, index=demand_data.index + np.ones((ndims, ndims)), + columns=demand_data.index, # type: ignore + index=demand_data.index, # type: ignore ) # @@ -45,12 +81,12 @@ def test_combined_workflow(): # damage_db = pelicun.file_io.load_data( - 'pelicun/tests/validation/2/data/additional_damage_db.csv', + 'pelicun/tests/validation/v2/data/additional_damage_db.csv', reindex=False, unit_conversion_factors=asmnt.unit_conversion_factors, ) consequences = pelicun.file_io.load_data( - 'pelicun/tests/validation/2/data/additional_consequences.csv', + 'pelicun/tests/validation/v2/data/additional_consequences.csv', reindex=False, unit_conversion_factors=asmnt.unit_conversion_factors, ) @@ -60,7 +96,7 @@ def test_combined_workflow(): # loss_functions = pelicun.file_io.load_data( - 'pelicun/tests/validation/2/data/additional_loss_functions.csv', + 'pelicun/tests/validation/v2/data/additional_loss_functions.csv', reindex=False, unit_conversion_factors=asmnt.unit_conversion_factors, ) @@ -75,34 +111,31 @@ def test_combined_workflow(): ) # Generate samples - asmnt.demand.generate_sample({"SampleSize": sample_size}) - - def add_more_edps(): - """ - Adds SA_1.13 and residual drift to the demand sample. + asmnt.demand.generate_sample({'SampleSize': sample_size}) - """ + def add_more_edps() -> None: + """Adds SA_1.13 and residual drift to the demand sample.""" # Add residual drift and Sa - demand_sample, demand_units = asmnt.demand.save_sample(save_units=True) + demand_sample = asmnt.demand.save_sample() # RIDs are all fixed for testing. - RID = pd.concat( + rid = pd.concat( [ pd.DataFrame( - np.full(demand_sample['PID'].shape, 0.0050), - index=demand_sample['PID'].index, - columns=demand_sample['PID'].columns, + np.full(demand_sample['PID'].shape, 0.0050), # type: ignore + index=demand_sample['PID'].index, # type: ignore + columns=demand_sample['PID'].columns, # type: ignore ) ], axis=1, keys=['RID'], ) - demand_sample_ext = pd.concat([demand_sample, RID], axis=1) + demand_sample_ext = pd.concat([demand_sample, rid], axis=1) # type: ignore - demand_sample_ext[('SA_1.13', 0, 1)] = 1.50 + demand_sample_ext['SA_1.13', 0, 1] = 1.50 # Add units to the data - demand_sample_ext.T.insert(0, 'Units', "") + demand_sample_ext.T.insert(0, 'Units', '') # PFA and SA are in "g" in this example, while PID and RID are "rad" demand_sample_ext.loc['Units', ['PFA', 'SA_1.13']] = 'g' @@ -121,7 +154,7 @@ def add_more_edps(): # Load component definitions cmp_marginals = pd.read_csv( - 'pelicun/tests/validation/2/data/CMP_marginals.csv', index_col=0 + 'pelicun/tests/validation/v2/data/CMP_marginals.csv', index_col=0 ) cmp_marginals['Blocks'] = cmp_marginals['Blocks'] asmnt.asset.load_cmp_model({'marginals': cmp_marginals}) @@ -129,16 +162,23 @@ def add_more_edps(): # Generate sample asmnt.asset.generate_cmp_sample(sample_size) + # # Used to test that the example works when an existing sample is + # # loaded. + # asmnt.asset.save_cmp_sample(filepath='/tmp/cmp_sample.csv', save_units=True) + # asmnt.asset.cmp_sample + # asmnt.asset.load_cmp_sample(filepath='/tmp/cmp_sample.csv') + # asmnt.asset.cmp_sample + # # Damage # - cmp_set = asmnt.asset.list_unique_component_ids(as_set=True) + cmp_set = set(asmnt.asset.list_unique_component_ids()) # Load the models into pelicun asmnt.damage.load_model_parameters( [ - damage_db, + damage_db, # type: ignore 'PelicunDefault/damage_DB_FEMA_P58_2nd.csv', ], cmp_set, @@ -146,8 +186,8 @@ def add_more_edps(): # Prescribe the damage process dmg_process = { - "1_collapse": {"DS1": "ALL_NA"}, - "2_excessiveRID": {"DS1": "irreparable_DS1"}, + '1_collapse': {'DS1': 'ALL_NA'}, + '2_excessiveRID': {'DS1': 'irreparable_DS1'}, } # Calculate damages @@ -158,6 +198,7 @@ def add_more_edps(): asmnt.damage.save_sample(f'{temp_dir}/out.csv') asmnt.damage.load_sample(f'{temp_dir}/out.csv') + assert asmnt.damage.ds_model.sample is not None asmnt.damage.ds_model.sample.mean() # @@ -177,9 +218,9 @@ def add_more_edps(): with pytest.warns(PelicunWarning): asmnt.loss.load_model_parameters( [ - consequences, - loss_functions, - "PelicunDefault/loss_repair_DB_FEMA_P58_2nd.csv", + consequences, # type: ignore + loss_functions, # type: ignore + 'PelicunDefault/loss_repair_DB_FEMA_P58_2nd.csv', ] ) diff --git a/pelicun/tools/DL_calculation.py b/pelicun/tools/DL_calculation.py index c716530c1..fc6fb9916 100644 --- a/pelicun/tools/DL_calculation.py +++ b/pelicun/tools/DL_calculation.py @@ -1,5 +1,4 @@ -# -*- coding: utf-8 -*- -# +# # noqa: N999 # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California # @@ -36,279 +35,89 @@ # # Contributors: # Adam Zsarnóczay +# John Vouvakis Manousakis -""" -This module provides the main functionality to run a pelicun -calculation from the command line. - -""" +"""Main functionality to run a pelicun calculation from the command line.""" from __future__ import annotations -from typing import Any -from time import gmtime -from time import strftime -import sys -import os -import json + import argparse +import json +import os +import sys from pathlib import Path - -import numpy as np -import pandas as pd +from time import gmtime, strftime +from typing import Hashable import colorama -from colorama import Fore -from colorama import Style - import jsonschema +import numpy as np +import pandas as pd +from colorama import Fore, Style from jsonschema import validate -import pelicun -from pelicun.auto import auto_populate -from pelicun.base import str2bool -from pelicun.base import convert_to_MultiIndex -from pelicun.base import convert_to_SimpleIndex -from pelicun.base import describe -from pelicun.base import EDP_to_demand_type from pelicun import base -from pelicun.file_io import load_data -from pelicun.assessment import Assessment - - -# pylint: disable=consider-using-namedtuple-or-dataclass -# pylint: disable=too-many-statements -# pylint: disable=too-many-nested-blocks -# pylint: disable=too-many-arguments -# pylint: disable=else-if-used -# pylint: disable=unused-argument - -# pd.set_option('display.max_rows', None) +from pelicun.assessment import DLCalculationAssessment +from pelicun.auto import auto_populate +from pelicun.base import ( + convert_to_MultiIndex, + convert_to_SimpleIndex, + describe, + get, + is_specified, + is_unspecified, + str2bool, + update, + update_vals, +) +from pelicun.pelicun_warnings import PelicunInvalidConfigError colorama.init() +sys.path.insert(0, Path(__file__).resolve().parent.absolute().as_posix()) -def get(d: dict[str, Any], path: str, default: Any | None = None) -> Any: +def log_msg(msg: str, color_codes: tuple[str, str] | None = None) -> None: """ - Retrieve a value from a nested dictionary using a path with '/' as - the separator. + Print a formatted log message with a timestamp. Parameters ---------- - d : dict - The dictionary to search. - path : str - The path to the desired value, with keys separated by '/'. - default : Any, optional - The value to return if the path is not found. Defaults to - None. - - Returns - ------- - Any - The value found at the specified path, or the default value if - the path is not found. - - Examples - -------- - >>> config = { - ... "DL": { - ... "Outputs": { - ... "Format": { - ... "JSON": "desired_value" - ... } - ... } - ... } - ... } - >>> get(config, '/DL/Outputs/Format/JSON', default='default_value') - 'desired_value' - >>> get(config, '/DL/Outputs/Format/XML', default='default_value') - 'default_value' - - """ - keys = path.strip('/').split('/') - current_dict = d - try: - for key in keys: - current_dict = current_dict[key] - return current_dict - except (KeyError, TypeError): - return default - - -def update( - d: dict[str, Any], path: str, value: Any, only_if_empty_or_none: bool = False -) -> None: - """ - Set a value in a nested dictionary using a path with '/' as the separator. + msg : str + The message to print. + color_codes : tuple, optional + Color codes for formatting the message. Default is None. - Parameters - ---------- - d : dict - The dictionary to update. - path : str - The path to the desired value, with keys separated by '/'. - value : Any - The value to set at the specified path. - only_if_empty_or_none : bool, optional - If True, only update the value if it is None or an empty - dictionary. Defaults to False. - - Examples - -------- - >>> d = {} - >>> update(d, 'x/y/z', 1) - >>> d - {'x': {'y': {'z': 1}}} - - >>> update(d, 'x/y/z', 2, only_if_empty_or_none=True) - >>> d - {'x': {'y': {'z': 1}}} # value remains 1 since it is not empty or None - - >>> update(d, 'x/y/z', 2) - >>> d - {'x': {'y': {'z': 2}}} # value is updated to 2 """ - - keys = path.strip('/').split('/') - current_dict = d - for key in keys[:-1]: - if key not in current_dict or not isinstance(current_dict[key], dict): - current_dict[key] = {} - current_dict = current_dict[key] - if only_if_empty_or_none: - if is_unspecified(current_dict, keys[-1]): - current_dict[keys[-1]] = value + if color_codes: + cpref, csuff = color_codes + print( # noqa: T201 + f'{strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())} ' + f'{cpref}' + f'{msg}' + f'{csuff}' + ) else: - current_dict[keys[-1]] = value - - -def is_unspecified(d: dict[str, Any], path: str) -> bool: - """ - Check if a value in a nested dictionary is either non-existent, - None, NaN, or an empty dictionary or list. - - Parameters - ---------- - d : dict - The dictionary to search. - path : str - The path to the desired value, with keys separated by '/'. - - Returns - ------- - bool - True if the value is non-existent, None, or an empty - dictionary or list. False otherwise. - - Examples - -------- - >>> config = { - ... "DL": { - ... "Outputs": { - ... "Format": { - ... "JSON": "desired_value", - ... "EmptyDict": {} - ... } - ... } - ... } - ... } - >>> is_none_or_empty(config, '/DL/Outputs/Format/JSON') - False - >>> is_none_or_empty(config, '/DL/Outputs/Format/XML') - True - >>> is_none_or_empty(config, '/DL/Outputs/Format/EmptyDict') - True - - """ - value = get(d, path, default=None) - if value is None: - return True - if pd.isna(value): - return True - if value == {}: - return True - if value == []: - return True - return False - - -def log_msg(msg): - """ - Prints a formatted string to stdout in the form of a log. Includes - a timestamp. - - Parameters - ---------- - msg: str - The message to be printed. - - """ - formatted_msg = f'{strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())} {msg}' - - print(formatted_msg) - - -sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) - -idx = pd.IndexSlice - -# TODO: separate Damage Processes for -# HAZUS Earthquake - Buildings and - Transportation -# TODO: Loss map for HAZUS EQ Transportation + print(f'{strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())} {msg}') # noqa: T201 -damage_processes = { - 'FEMA P-58': { - "1_excessive.coll.DEM": {"DS1": "collapse_DS1"}, - "2_collapse": {"DS1": "ALL_NA"}, - "3_excessiveRID": {"DS1": "irreparable_DS1"}, - }, - # TODO: expand with ground failure logic - 'Hazus Earthquake': { - "1_STR": {"DS5": "collapse_DS1"}, - "2_LF": {"DS5": "collapse_DS1"}, - "3_excessive.coll.DEM": {"DS1": "collapse_DS1"}, - "4_collapse": {"DS1": "ALL_NA"}, - "5_excessiveRID": {"DS1": "irreparable_DS1"}, - }, - 'Hazus Hurricane': {}, -} - -default_DBs = { - 'fragility': { - 'FEMA P-58': 'damage_DB_FEMA_P58_2nd.csv', - 'Hazus Earthquake - Buildings': 'damage_DB_Hazus_EQ_bldg.csv', - 'Hazus Earthquake - Stories': 'damage_DB_Hazus_EQ_story.csv', - 'Hazus Earthquake - Transportation': 'damage_DB_Hazus_EQ_trnsp.csv', - 'Hazus Earthquake - Water': 'damage_DB_Hazus_EQ_water.csv', - 'Hazus Hurricane': 'damage_DB_SimCenter_Hazus_HU_bldg.csv', - }, - 'repair': { - 'FEMA P-58': 'loss_repair_DB_FEMA_P58_2nd.csv', - 'Hazus Earthquake - Buildings': 'loss_repair_DB_Hazus_EQ_bldg.csv', - 'Hazus Earthquake - Stories': 'loss_repair_DB_Hazus_EQ_story.csv', - 'Hazus Earthquake - Transportation': 'loss_repair_DB_Hazus_EQ_trnsp.csv', - 'Hazus Hurricane': 'loss_repair_DB_SimCenter_Hazus_HU_bldg.csv', - }, -} # list of output files help perform safe initialization of output dir -output_files = [ - "DEM_sample.zip", - "DEM_stats.csv", - "CMP_sample.zip", - "CMP_stats.csv", - "DMG_sample.zip", - "DMG_stats.csv", - "DMG_grp.zip", - "DMG_grp_stats.csv", - "DV_repair_sample.zip", - "DV_repair_stats.csv", - "DV_repair_grp.zip", - "DV_repair_grp_stats.csv", - "DV_repair_agg.zip", - "DV_repair_agg_stats.csv", - "DL_summary.csv", - "DL_summary_stats.csv", +known_output_files = [ + 'DEM_sample.zip', + 'DEM_stats.csv', + 'CMP_sample.zip', + 'CMP_stats.csv', + 'DMG_sample.zip', + 'DMG_stats.csv', + 'DMG_grp.zip', + 'DMG_grp_stats.csv', + 'DV_repair_sample.zip', + 'DV_repair_stats.csv', + 'DV_repair_grp.zip', + 'DV_repair_grp_stats.csv', + 'DV_repair_agg.zip', + 'DV_repair_agg_stats.csv', + 'DL_summary.csv', + 'DL_summary_stats.csv', ] full_out_config = { @@ -367,13 +176,13 @@ def log_msg(msg): } -def convert_df_to_dict(df, axis=1): +def convert_df_to_dict(data: pd.DataFrame | pd.Series, axis: int = 1) -> dict: """ Convert a pandas DataFrame to a dictionary. Parameters ---------- - df : pd.DataFrame + data : pd.DataFrame The DataFrame to be converted. axis : int, optional The axis to consider for the conversion. @@ -403,31 +212,31 @@ def convert_df_to_dict(df, axis=1): as values. """ - - out_dict = {} + out_dict: dict[Hashable, object] = {} if axis == 1: - df_in = df + df_in = data elif axis == 0: - df_in = df.T + df_in = data.T else: - raise ValueError('`axis` must be `0` or `1`') + msg = '`axis` must be `0` or `1`' + raise ValueError(msg) - MI = df_in.columns + multiindex = df_in.columns - for label in MI.unique(level=0): + for label in multiindex.unique(level=0): out_dict.update({label: np.nan}) sub_df = df_in[label] skip_sub = True - if MI.nlevels > 1: + if multiindex.nlevels > 1: skip_sub = False - if isinstance(sub_df, pd.Series): - skip_sub = True - elif (len(sub_df.columns) == 1) and (sub_df.columns[0] == ""): + if isinstance(sub_df, pd.Series) or ( + (len(sub_df.columns) == 1) and (sub_df.columns[0] == '') # noqa: PLC1901 + ): skip_sub = True if not skip_sub: @@ -443,256 +252,478 @@ def convert_df_to_dict(df, axis=1): return out_dict -def add_units(raw_demands, length_unit): +def run_pelicun( + config_path: str, + demand_file: str, + output_path: str | None, + realizations: int, + auto_script_path: str | None, + custom_model_dir: str | None, + output_format: list | None, + *, + detailed_results: bool, + coupled_edp: bool, +) -> None: """ - Add units to demand columns in a DataFrame. + Use settings in the config JSON to prepare and run a Pelicun calculation. Parameters ---------- - raw_demands : pd.DataFrame - The raw demand data to which units will be added. - length_unit : str - The unit of length to be used (e.g., 'in' for inches). - - Returns - ------- - pd.DataFrame - The DataFrame with units added to the appropriate demand columns. + config_path: string + Path pointing to the location of the JSON configuration file. + demand_file: string + Path pointing to the location of a CSV file with the demand data. + output_path: string, optional + Path pointing to the location where results shall be saved. + realizations: int, optional + Number of realizations to generate. + auto_script_path: string, optional + Path pointing to the location of a Python script with an auto_populate + method that automatically creates the performance model using data + provided in the AIM JSON file. + custom_model_dir: string, optional + Path pointing to a directory with files that define user-provided model + parameters for a customized damage and loss assessment. + output_format: list, optional. + Type of output format, JSON or CSV. + Valid options: ['csv', 'json'], ['csv'], ['json'], [], None + detailed_results: bool, optional + If False, only the main statistics are saved. + coupled_edp: bool, optional + If True, EDPs are not resampled and processed in order. """ - demands = raw_demands.T - - demands.insert(0, "Units", np.nan) - - if length_unit == 'in': - length_unit = 'inch' + # Initial setup ----------------------------------------------------------- - demands = convert_to_MultiIndex(demands, axis=0).sort_index(axis=0).T + # get the absolute path to the config file + config_path_p = Path(config_path).resolve() - if demands.columns.nlevels == 4: - DEM_level = 1 + # If the output path was not specified, results are saved in the + # directory of the input file. + if output_path is None: + output_path_p = config_path_p.parents[0] else: - DEM_level = 0 - - # drop demands with no EDP type identified - demands.drop( - demands.columns[demands.columns.get_level_values(DEM_level) == ''], - axis=1, - inplace=True, + output_path_p = Path(output_path).resolve() + # create the directory if it does not exist + if not output_path_p.exists(): + output_path_p.mkdir(parents=True) + + # parse the config file + config = _parse_config_file( + config_path_p, + output_path_p, + Path(auto_script_path).resolve() if auto_script_path is not None else None, + demand_file, + realizations, + output_format, + coupled_edp=coupled_edp, + detailed_results=detailed_results, ) - # assign units - demand_cols = demands.columns.get_level_values(DEM_level) - - # remove additional info from demand names - demand_cols = [d.split('_')[0] for d in demand_cols] + # List to keep track of the generated output files. + out_files: list[str] = [] + + _remove_existing_files(output_path_p, known_output_files) + + # Run the assessment + assessment = DLCalculationAssessment(config_options=get(config, 'DL/Options')) + + assessment.calculate_demand( + demand_path=Path(get(config, 'DL/Demands/DemandFilePath')).resolve(), + collapse_limits=get(config, 'DL/Demands/CollapseLimits', default=None), + length_unit=get(config, 'GeneralInformation/units/length', default=None), + demand_calibration=get(config, 'DL/Demands/Calibration', default=None), + sample_size=get(config, 'DL/Options/Sampling/SampleSize'), + demand_cloning=get(config, 'DL/Demands/DemandCloning', default=None), + residual_drift_inference=get( + config, 'DL/Demands/InferResidualDrift', default=None + ), + coupled_demands=get(config, 'DL/Demands/CoupledDemands', default=False), + ) - # acceleration - acc_EDPs = ['PFA', 'PGA', 'SA'] - EDP_mask = np.isin(demand_cols, acc_EDPs) + if is_specified(config, 'DL/Asset'): + assessment.calculate_asset( + num_stories=get(config, 'DL/Asset/NumberOfStories', default=None), + component_assignment_file=get( + config, 'DL/Asset/ComponentAssignmentFile', default=None + ), + collapse_fragility_demand_type=get( + config, 'DL/Damage/CollapseFragility/DemandType', default=None + ), + component_sample_file=get( + config, 'DL/Asset/ComponentSampleFile', default=None + ), + add_irreparable_damage_columns=get( + config, 'DL/Damage/IrreparableDamage', default=False + ), + ) - if np.any(EDP_mask): - demands.iloc[0, EDP_mask] = length_unit + 'ps2' + if is_specified(config, 'DL/Damage'): + assessment.calculate_damage( + length_unit=get(config, 'GeneralInformation/units/length'), + component_database=get(config, 'DL/Asset/ComponentDatabase'), + component_database_path=get( + config, 'DL/Asset/ComponentDatabasePath', default=None + ), + collapse_fragility=get( + config, 'DL/Damage/CollapseFragility', default=None + ), + irreparable_damage=get( + config, 'DL/Damage/IrreparableDamage', default=None + ), + damage_process_approach=get( + config, 'DL/Damage/DamageProcess', default=None + ), + damage_process_file_path=get( + config, 'DL/Damage/DamageProcessFilePath', default=None + ), + custom_model_dir=custom_model_dir, + scaling_specification=get(config, 'DL/Damage/ScalingSpecification'), + is_for_water_network_assessment=is_specified( + config, 'DL/Asset/ComponentDatabase/Water' + ), + ) - # speed - speed_EDPs = ['PFV', 'PWS', 'PGV', 'SV'] - EDP_mask = np.isin(demand_cols, speed_EDPs) + if is_unspecified(config, 'DL/Losses/Repair'): + agg_repair = None + else: + # Currently we only support `Repair` consequences. + # We will need to make changes here when we start to include + # more consequences. + + agg_repair, _ = assessment.calculate_loss( + loss_map_approach=get(config, 'DL/Losses/Repair/MapApproach'), + occupancy_type=get(config, 'DL/Asset/OccupancyType'), + consequence_database=get(config, 'DL/Losses/Repair/ConsequenceDatabase'), + consequence_database_path=get( + config, 'DL/Losses/Repair/ConsequenceDatabasePath' + ), + custom_model_dir=custom_model_dir, + damage_process_approach=get( + config, 'DL/Damage/DamageProcess', default='User Defined' + ), + replacement_cost_parameters=get( + config, 'DL/Losses/Repair/ReplacementCost' + ), + replacement_time_parameters=get( + config, 'DL/Losses/Repair/ReplacementTime' + ), + replacement_carbon_parameters=get( + config, 'DL/Losses/Repair/ReplacementCarbon' + ), + replacement_energy_parameters=get( + config, 'DL/Losses/Repair/ReplacementEnergy' + ), + loss_map_path=get(config, 'DL/Losses/Repair/MapFilePath'), + decision_variables=_parse_decision_variables(config), + ) - if np.any(EDP_mask): - demands.iloc[0, EDP_mask] = length_unit + 'ps' + summary, summary_stats = _result_summary(assessment, agg_repair) + + # Save the results into files + + if is_specified(config, 'DL/Outputs/Demand'): + output_config = get(config, 'DL/Outputs/Demand') + _demand_save(output_config, assessment, output_path_p, out_files) + + if is_specified(config, 'DL/Outputs/Asset'): + output_config = get(config, 'DL/Outputs/Asset') + _asset_save( + output_config, + assessment, + output_path_p, + out_files, + aggregate_colocated=get( + config, + 'DL/Outputs/Settings/AggregateColocatedComponentResults', + default=False, + ), + ) - # displacement - disp_EDPs = ['PFD', 'PIH', 'SD', 'PGD'] - EDP_mask = np.isin(demand_cols, disp_EDPs) + if is_specified(config, 'DL/Outputs/Damage'): + output_config = get(config, 'DL/Outputs/Damage') + _damage_save( + output_config, + assessment, + output_path_p, + out_files, + aggregate_colocated=get( + config, + 'DL/Outputs/Settings/AggregateColocatedComponentResults', + default=False, + ), + condense_ds=get( + config, + 'DL/Outputs/Settings/CondenseDS', + default=False, + ), + ) - if np.any(EDP_mask): - demands.iloc[0, EDP_mask] = length_unit + if is_specified(config, 'DL/Outputs/Loss/Repair'): + output_config = get(config, 'DL/Outputs/Loss/Repair') + assert agg_repair is not None + _loss_save( + output_config, + assessment, + output_path_p, + out_files, + agg_repair, + aggregate_colocated=get( + config, + 'DL/Outputs/Settings/AggregateColocatedComponentResults', + default=False, + ), + ) + _summary_save(summary, summary_stats, output_path_p, out_files) + _create_json_files_if_requested(config, out_files, output_path_p) + _remove_csv_files_if_not_requested(config, out_files, output_path_p) - # drift ratio - rot_EDPs = ['PID', 'PRD', 'DWD', 'RDR', 'PMD', 'RID'] - EDP_mask = np.isin(demand_cols, rot_EDPs) - if np.any(EDP_mask): - demands.iloc[0, EDP_mask] = 'unitless' +def _parse_decision_variables(config: dict) -> tuple[str, ...]: + """ + Parse decision variables from the config file. - # convert back to simple header and return the DF - return convert_to_SimpleIndex(demands, axis=1) + Parameters + ---------- + config : dict + The configuration dictionary. + Returns + ------- + list + List of decision variables. -def run_pelicun( - config_path, - demand_file, - output_path, - coupled_EDP, - realizations, - auto_script_path, - detailed_results, - regional, - output_format, - custom_model_dir, - color_warnings, - **kwargs, -): """ - Use settings in the config JSON to prepare and run a Pelicun calculation. + decision_variables: list[str] = [] + if get(config, 'DL/Losses/Repair/DecisionVariables', default=False) is not False: + for dv_i, dv_status in get( + config, 'DL/Losses/Repair/DecisionVariables' + ).items(): + if dv_status is True: + decision_variables.append(dv_i) + return tuple(decision_variables) + + +def _remove_csv_files_if_not_requested( + config: dict, out_files: list[str], output_path: Path +) -> None: + """ + Remove CSV files if not requested in config. Parameters ---------- - config_path: string - Path pointing to the location of the JSON configuration file. - demand_file: string - Path pointing to the location of a CSV file with the demand data. - output_path: string, optional - Path pointing to the location where results shall be saved. - coupled_EDP: bool, optional - If True, EDPs are not resampled and processed in order. - realizations: int, optional - Number of realizations to generate. - auto_script_path: string, optional - Path pointing to the location of a Python script with an auto_populate - method that automatically creates the performance model using data - provided in the AIM JSON file. - detailed_results: bool, optional - If False, only the main statistics are saved. - regional: bool - Currently unused. - output_format: str - Type of output format, JSON or CSV. - custom_model_dir: string, optional - Path pointing to a directory with files that define user-provided model - parameters for a customized damage and loss assessment. - color_warnings: bool, optional - If True, warnings are printed in red on the console. If output - is redirected to a file, it will contain ANSI codes. When - viewed on the console with `cat`, `less`, or similar utilites, - the color will be shown. + config : dict + Configuration dictionary. + out_files : list + List of output file names. + output_path : Path + Path to the output directory. + """ + # Don't proceed if CSV files were requested. + if get(config, 'DL/Outputs/Format/CSV', default=False) is True: + return - Returns - ------- - bool - 0 if the calculation was ran successfully, or -1 otherwise. + for filename in out_files: + # keep the DL_summary and DL_summary_stats files + if 'DL_summary' in filename: + continue + Path(output_path / filename).unlink() + +def _summary_save( + summary: pd.DataFrame, + summary_stats: pd.DataFrame, + output_path: Path, + out_files: list[str], +) -> None: """ + Save summary results to CSV files. - log_msg('First line of DL_calculation') + Parameters + ---------- + summary : pd.DataFrame + Summary DataFrame. + summary_stats : pd.DataFrame + Summary statistics DataFrame. + output_path : Path + Path to the output directory. + out_files : list + List of output file names. - # Initial setup ----------------------------------------------------------- + """ + # save summary sample + if summary is not None: + summary.to_csv(output_path / 'DL_summary.csv', index_label='#') + out_files.append('DL_summary.csv') - # color warnings - cpref, csuff = _get_color_codes(color_warnings) + # save summary statistics + if summary_stats is not None: + summary_stats.to_csv(output_path / 'DL_summary_stats.csv') + out_files.append('DL_summary_stats.csv') + + +def _parse_config_file( # noqa: C901 + config_path: Path, + output_path: Path, + auto_script_path: Path | None, + demand_file: str, + realizations: int, + output_format: list | None, + *, + coupled_edp: bool, + detailed_results: bool, +) -> dict[str, object]: + """ + Parse and validate the config file for Pelicun. - # get the absolute path to the config file - config_path = Path(config_path).resolve() + Parameters + ---------- + config_path : str + Path to the configuration file. + output_path : Path + Directory for output files. + auto_script_path : str + Path to the auto-generation script. + demand_file : str + Path to the demand data file. + realizations : int + Number of realizations. + coupled_EDP : bool + Whether to consider coupled EDPs. + detailed_results : bool + Whether to generate detailed results. + output_format : str + Output format (CSV, JSON). - # If the output path was not specified, results are saved in the directory - # of the input file. - if output_path is None: - output_path = config_path.parents[0] - else: - output_path = Path(output_path) + Returns + ------- + dict + Parsed and validated configuration. - # Initialize the array that we'll use to collect the output file names - out_files = _remove_existing_files(output_path) + Raises + ------ + PelicunInvalidConfigError + If the provided config file does not conform to the schema or + there are issues with the specified values. + """ # open the config file and parse it - with open(config_path, 'r', encoding='utf-8') as f: + with Path(config_path).open(encoding='utf-8') as f: config = json.load(f) # load the schema - with open( - f'{base.pelicun_path}/settings/schema.json', 'r', encoding='utf-8' + with Path(f'{base.pelicun_path}/settings/input_schema.json').open( + encoding='utf-8' ) as f: schema = json.load(f) - # Validate the configuration against the schema - validate(instance=config, schema=schema) + # add the demand file to the DL if needed + if is_specified(config, 'DL'): + if is_unspecified(config, 'DL/Demands/DemandFilePath'): + update(config, '/DL/Demands/DemandFilePath', demand_file) - # f"{config['commonFileDir']}/CustomDLModels/" - custom_dl_file_path = custom_model_dir + # Validate the configuration against the schema + try: + validate(instance=config, schema=schema) + except jsonschema.exceptions.ValidationError as exc: + msg = 'The provided config file does not conform to the schema.' + raise PelicunInvalidConfigError(msg) from exc if is_unspecified(config, 'DL'): - log_msg("Damage and Loss configuration missing from config file. ") - - if auto_script_path is not None: - log_msg("Trying to auto-populate") + log_msg('Damage and Loss configuration missing from config file. ') - config_ap, CMP = auto_populate(config, auto_script_path) + if auto_script_path is None: + msg = 'No `DL` entry in config file.' + raise PelicunInvalidConfigError(msg) - if is_unspecified(config_ap, 'DL'): + log_msg('Trying to auto-populate') - log_msg( - "The prescribed auto-population script failed to identify " - "a valid damage and loss configuration for this asset. " - "Terminating analysis." - ) + # Add the demandFile to the config dict to allow demand dependent auto-population + update(config, '/DL/Demands/DemandFilePath', demand_file) + update(config, '/DL/Demands/SampleSize', str(realizations)) - return -1 + config_ap, comp = auto_populate(config, auto_script_path) - # add the demand information - update(config_ap, '/DL/Demands/DemandFilePath', demand_file) - update(config_ap, '/DL/Demands/SampleSize', str(realizations)) + if is_unspecified(config_ap, 'DL'): + msg = ( + 'No `DL` entry in config file, and ' + 'the prescribed auto-population script failed to identify ' + 'a valid damage and loss configuration for this asset. ' + ) + raise PelicunInvalidConfigError(msg) - if coupled_EDP is True: - update(config_ap, 'DL/Demands/CoupledDemands', True) + # look for possibly specified assessment options + try: + assessment_options = config['Applications']['DL']['ApplicationData'][ + 'Options' + ] + except KeyError: + assessment_options = None + + if assessment_options: + # extend options defined via the auto-population script to + # include those in the original `config` + config_ap['Applications']['DL']['ApplicationData'].pop('Options') + update_vals( + config_ap['DL']['Options'], + assessment_options, + "config_ap['DL']['Options']", + 'assessment_options', + ) - else: - update( - config_ap, - 'DL/Demands/Calibration', - {"ALL": {"DistributionFamily": "lognormal"}}, - ) + # add the demand information + update(config_ap, '/DL/Demands/DemandFilePath', demand_file) + update(config_ap, '/DL/Demands/SampleSize', str(realizations)) - # save the component data - CMP.to_csv(output_path / 'CMP_QNT.csv') + if coupled_edp is True: + update(config_ap, 'DL/Demands/CoupledDemands', value=True) - # update the config file with the location + else: update( config_ap, - 'DL/Asset/ComponentAssignmentFile', - str(output_path / 'CMP_QNT.csv'), + 'DL/Demands/Calibration', + {'ALL': {'DistributionFamily': 'lognormal'}}, ) - # if detailed results are not requested, add a lean output config - if detailed_results is False: - update(config_ap, 'DL/Outputs', regional_out_config) - else: - update(config_ap, 'DL/Outputs', full_out_config) - # add output settings from regional output config - if is_unspecified(config_ap, 'DL/Outputs/Settings'): - update(config_ap, 'DL/Outputs/Settings', {}) + # save the component data + comp.to_csv(output_path / 'CMP_QNT.csv') - config_ap['DL']['Outputs']['Settings'].update( - regional_out_config['Settings'] - ) + # update the config file with the location + update( + config_ap, + 'DL/Asset/ComponentAssignmentFile', + str(output_path / 'CMP_QNT.csv'), + ) - # save the extended config to a file - config_ap_path = Path(config_path.stem + '_ap.json').resolve() + # if detailed results are not requested, add a lean output config + if detailed_results is False: + update(config_ap, 'DL/Outputs', regional_out_config) + else: + update(config_ap, 'DL/Outputs', full_out_config) + # add output settings from regional output config + if is_unspecified(config_ap, 'DL/Outputs/Settings'): + update(config_ap, 'DL/Outputs/Settings', {}) - with open(config_ap_path, 'w', encoding='utf-8') as f: - json.dump(config_ap, f, indent=2) + config_ap['DL']['Outputs']['Settings'].update( + regional_out_config['Settings'] + ) - update(config, 'DL', get(config_ap, 'DL')) + # save the extended config to a file + config_ap_path = Path(config_path.stem + '_ap.json').resolve() - else: - log_msg("Terminating analysis.") + with Path(config_ap_path).open('w', encoding='utf-8') as f: + json.dump(config_ap, f, indent=2) - return -1 + update(config, 'DL', get(config_ap, 'DL')) - # - # sample size: backwards compatibility - # - sample_size_str = ( - # expected location - get(config, 'Options/Sampling/SampleSize') - ) + # sample size + sample_size_str = get(config, 'DL/Options/Sampling/SampleSize') if not sample_size_str: - # try previous location sample_size_str = get(config, 'DL/Demands/SampleSize') - if not sample_size_str: - # give up - print('Sample size not provided in config file. Terminating analysis.') - return -1 - sample_size = int(sample_size_str) + if not sample_size_str: + msg = 'Sample size not provided in config file.' + raise PelicunInvalidConfigError(msg) + update(config, 'DL/Options/Sampling/SampleSize', int(sample_size_str)) # provide all outputs if the files are not specified if is_unspecified(config, 'DL/Outputs'): @@ -702,7 +733,8 @@ def run_pelicun( if is_unspecified(config, 'DL/Outputs/Format'): update(config, 'DL/Outputs/Format', {'CSV': True, 'JSON': False}) - # override file format specification if the output_format is provided + # override file format specification if the output_format is + # provided if output_format is not None: update( config, @@ -717,409 +749,237 @@ def run_pelicun( if is_unspecified(config, 'DL/Outputs/Settings'): update(config, 'DL/Outputs/Settings', pbe_settings) - if is_unspecified(config, 'DL/Asset'): - log_msg("Asset configuration missing. Terminating analysis.") - return -1 - if is_unspecified(config, 'DL/Demands'): - log_msg("Demand configuration missing. Terminating analysis.") - return -1 + msg = 'Demand configuration missing.' + raise PelicunInvalidConfigError(msg) - # get the length unit from the config file - try: - length_unit = get(config, 'GeneralInformation/units/length') - except KeyError: - log_msg( - "No default length unit provided in the input file. " - "Terminating analysis. " - ) - return -1 + if is_unspecified(config, 'DL/Asset'): + msg = 'Asset configuration missing.' + raise PelicunInvalidConfigError(msg) - # initialize the Pelicun Assessement - update(config, 'DL/Options/LogFile', 'pelicun_log.txt') - update(config, 'DL/Options/Verbose', True) + update( + config, + 'DL/Options/LogFile', + 'pelicun_log.txt', + only_if_empty_or_none=True, + ) + update( + config, + 'DL/Options/Verbose', + value=True, + only_if_empty_or_none=True, + ) - # If the user did not prescribe anything for ListAllDamageStates, + # if the user did not prescribe anything for ListAllDamageStates, # then use True as default for DL_calculations regardless of what # the Pelicun default is. update( - config, 'DL/Options/ListAllDamageStates', True, only_if_empty_or_none=True + config, + 'DL/Options/ListAllDamageStates', + value=True, + only_if_empty_or_none=True, ) - PAL = Assessment(get(config, 'DL/Options')) + # if the demand file location is not specified in the config file + # assume there is a `response.csv` file next to the config file. + update( + config, + 'DL/Demands/DemandFilePath', + config_path.parent / 'response.csv', + only_if_empty_or_none=True, + ) - # Demand Assessment ----------------------------------------------------------- + # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + # backwards-compatibility for v3.2 and earlier | remove after v4.0 + if get(config, 'DL/Losses/BldgRepair', default=False): + update(config, 'DL/Losses/Repair', get(config, 'DL/Losses/BldgRepair')) + if get(config, 'DL/Outputs/Loss/BldgRepair', default=False): + update( + config, + 'DL/Outputs/Loss/Repair', + get(config, 'DL/Outputs/Loss/BldgRepair'), + ) + # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - _demand(config, config_path, length_unit, PAL, sample_size) + # Cast NumberOfStories to int + if is_specified(config, 'DL/Asset/NumberOfStories'): + update( + config, + 'DL/Asset/NumberOfStories', + int(get(config, 'DL/Asset/NumberOfStories')), + ) - # if requested, save demand results - if not is_unspecified(config, 'DL/Outputs/Demand'): - demand_sample = _demand_save(config, PAL, output_path, out_files) - else: - demand_sample, _ = PAL.demand.save_sample(save_units=True) + # Ensure `DL/Demands/InferResidualDrift` contains a `method` + if is_specified(config, 'DL/Demands/InferResidualDrift') and is_unspecified( + config, 'DL/Demands/InferResidualDrift/method' + ): + msg = 'No method is specified in residual drift inference configuration.' + raise PelicunInvalidConfigError(msg) + + # Ensure `DL/Damage/CollapseFragility` contains all required keys. + if is_specified(config, 'DL/Damage/CollapseFragility'): + for thing in ('CapacityDistribution', 'CapacityMedian', 'Theta_1'): + if is_unspecified(config, f'DL/Damage/CollapseFragility/{thing}'): + msg = ( + f'`{thing}` is missing from DL/Damage/CollapseFragility' + f' in the configuration file.' + ) + raise PelicunInvalidConfigError(msg) + + # Ensure `DL/Damage/IrreparableDamage` contains all required keys. + if is_specified(config, 'DL/Damage/IrreparableDamage'): + for thing in ('DriftCapacityMedian', 'DriftCapacityLogStd'): + if is_unspecified(config, f'DL/Damage/IrreparableDamage/{thing}'): + msg = ( + f'`{thing}` is missing from DL/Damage/IrreparableDamage' + f' in the configuration file.' + ) + raise PelicunInvalidConfigError(msg) - # Asset Definition ------------------------------------------------------------ + # If the damage process approach is `User Defined` there needs to + # be a damage process file path. + if get(config, 'DL/Damage/DamageProcess') == 'User Defined' and is_unspecified( + config, 'DL/Damage/DamageProcessFilePath' + ): + msg = ( + 'When `DL/Damage/DamageProcess` is set to `User Defined`, ' + 'a path needs to be specified under ' + '`DL/Damage/DamageProcessFilePath`.' + ) + raise PelicunInvalidConfigError(msg) - # set the number of stories - cmp_marginals = _asset(config, PAL, demand_sample, cpref, csuff) + # Getting results requires running the calculations. + if is_specified(config, 'DL/Outputs/Asset') and is_unspecified( + config, 'DL/Asset' + ): + msg = ( + 'No asset data specified in config file. ' + 'Cannot generate asset model outputs.' + ) + raise PelicunInvalidConfigError(msg) - # if requested, save asset model results - if get(config, 'DL/Outputs/Asset', default=False): - _asset_save(PAL, config, output_path, out_files) + if is_specified(config, 'DL/Outputs/Damage') and is_unspecified( + config, 'DL/Damage' + ): + msg = ( + 'No damage data specified in config file. ' + 'Cannot generate damage model outputs.' + ) + raise PelicunInvalidConfigError(msg) - # Damage Assessment ----------------------------------------------------------- + if is_specified(config, 'DL/Outputs/Loss') and is_unspecified( + config, 'DL/Losses' + ): + msg = ( + 'No loss data specified in config file. ' + 'Cannot generate loss model outputs.' + ) + raise PelicunInvalidConfigError(msg) - # if a damage assessment is requested - if not is_unspecified(config, 'DL/Damage'): - # load the fragility information - try: + # Ensure only one of `component_assignment_file` or + # `component_sample_file` is provided. + if is_specified(config, 'DL/Asset'): + if ( + (get(config, 'DL/Asset/ComponentAssignmentFile') is None) + and (get(config, 'DL/Asset/ComponentSampleFile') is None) + ) or ( + (get(config, 'DL/Asset/ComponentAssignmentFile') is not None) + and (get(config, 'DL/Asset/ComponentSampleFile') is not None) + ): + msg = ( + 'In the asset model configuration, it is ' + 'required to specify one of `component_assignment_file` ' + 'or `component_sample_file`, but not both.' + ) + raise PelicunInvalidConfigError(msg) - _damage(config, custom_dl_file_path, PAL, cmp_marginals, length_unit) + return config - # if requested, save damage results - if not is_unspecified(config, 'DL/Outputs/Damage'): - damage_sample = _damage_save(PAL, config, output_path, out_files) - else: - damage_sample, _ = PAL.damage.save_sample(save_units=True) - except ValueError: - return -1 - else: - damage_sample, _ = PAL.damage.save_sample(save_units=True) +def _create_json_files_if_requested( + config: dict, out_files: list[str], output_path: Path +) -> None: + """ + Create JSON files if requested in the config. - # Loss Assessment ----------------------------------------------------------- + Parameters + ---------- + config : dict + Configuration dictionary. + out_files : list + List of output file names. + output_path : Path + Path to the output directory. - # if a loss assessment is requested - if not is_unspecified(config, 'DL/Losses'): - try: - agg_repair = _loss( - config, PAL, custom_dl_file_path, output_path, out_files - ) - except ValueError as e: - print(f'Exception occurred: {str(e)}. Terminating analysis') - return -1 - else: - agg_repair = None + """ + # If not requested, simply return + if get(config, 'DL/Outputs/Format/JSON', default=False) is False: + return - # Result Summary ----------------------------------------------------------- - - summary, summary_stats = _summary( - PAL, agg_repair, damage_sample, config, output_path, out_files - ) - - # save summary sample - summary.to_csv(output_path / "DL_summary.csv", index_label='#') - out_files.append('DL_summary.csv') - - # save summary statistics - summary_stats.to_csv(output_path / "DL_summary_stats.csv") - out_files.append('DL_summary_stats.csv') - - # create json outputs if needed - if get(config, 'DL/Outputs/Format/JSON') is True: - write_json_files(out_files, config, output_path) - - # remove csv outputs if they were not requested - if get(config, 'DL/Outputs/Format/CSV', default=False) is False: - for filename in out_files: - # keep the DL_summary and DL_summary_stats files - if 'DL_summary' in filename: - continue - os.remove(output_path / filename) - - return 0 - - -def write_json_files(out_files, config, output_path): - for filename in out_files: - filename_json = filename[:-3] + 'json' + for filename in out_files: + filename_json = filename[:-3] + 'json' if ( get(config, 'DL/Outputs/Settings/SimpleIndexInJSON', default=False) is True ): - df = pd.read_csv(output_path / filename, index_col=0) + data = pd.read_csv(output_path / filename, index_col=0) else: - df = convert_to_MultiIndex( + data = convert_to_MultiIndex( pd.read_csv(output_path / filename, index_col=0), axis=1 ) - if "Units" in df.index: + if 'Units' in data.index: df_units = convert_to_SimpleIndex( - df.loc['Units', :].to_frame().T, axis=1 + data.loc['Units', :].to_frame().T, # type: ignore + axis=1, ) - df.drop("Units", axis=0, inplace=True) + data = data.drop('Units', axis=0) - out_dict = convert_df_to_dict(df) + out_dict = convert_df_to_dict(data) out_dict.update( { - "Units": { - col: df_units.loc["Units", col] for col in df_units.columns + 'Units': { + col: df_units.loc['Units', col] for col in df_units.columns } } ) else: - out_dict = convert_df_to_dict(df) + out_dict = convert_df_to_dict(data) - with open(output_path / filename_json, 'w', encoding='utf-8') as f: + with Path(output_path / filename_json).open('w', encoding='utf-8') as f: json.dump(out_dict, f, indent=2) -def _damage_save(PAL, config, output_path, out_files): - damage_sample, damage_units = PAL.damage.save_sample(save_units=True) - damage_units = damage_units.to_frame().T - - if ( - get( - config, - 'DL/Outputs/Settings/AggregateColocatedComponentResults', - default=False, - ) - is True - ): - damage_units = damage_units.groupby(level=[0, 1, 2, 4], axis=1).first() - - damage_groupby_uid = damage_sample.groupby(level=[0, 1, 2, 4], axis=1) - - damage_sample = damage_groupby_uid.sum().mask( - damage_groupby_uid.count() == 0, np.nan - ) - - out_reqs = [ - out if val else "" for out, val in get(config, 'DL/Outputs/Damage').items() - ] - - if np.any( - np.isin( - [ - 'Sample', - 'Statistics', - 'GroupedSample', - 'GroupedStatistics', - ], - out_reqs, - ) - ): - if 'Sample' in out_reqs: - damage_sample_s = pd.concat([damage_sample, damage_units]) - - damage_sample_s = convert_to_SimpleIndex(damage_sample_s, axis=1) - damage_sample_s.to_csv( - output_path / "DMG_sample.zip", - index_label=damage_sample_s.columns.name, - compression={ - 'method': 'zip', - 'archive_name': 'DMG_sample.csv', - }, - ) - out_files.append('DMG_sample.zip') - - if 'Statistics' in out_reqs: - damage_stats = describe(damage_sample) - damage_stats = pd.concat([damage_stats, damage_units]) - - damage_stats = convert_to_SimpleIndex(damage_stats, axis=1) - damage_stats.to_csv( - output_path / "DMG_stats.csv", - index_label=damage_stats.columns.name, - ) - out_files.append('DMG_stats.csv') - - if np.any(np.isin(['GroupedSample', 'GroupedStatistics'], out_reqs)): - if ( - get( - config, - 'DL/Outputs/Settings/AggregateColocatedComponentResults', - default=False, - ) - is True - ): - damage_groupby = damage_sample.groupby(level=[0, 1, 3], axis=1) - - damage_units = damage_units.groupby(level=[0, 1, 3], axis=1).first() - - else: - damage_groupby = damage_sample.groupby(level=[0, 1, 4], axis=1) - - damage_units = damage_units.groupby(level=[0, 1, 4], axis=1).first() - - grp_damage = damage_groupby.sum().mask( - damage_groupby.count() == 0, np.nan - ) - - # if requested, condense DS output - if ( - get( - config, - 'DL/Outputs/Settings/CondenseDS', - default=False, - ) - is True - ): - # replace non-zero values with 1 - grp_damage = grp_damage.mask( - grp_damage.astype(np.float64).values > 0, 1 - ) - - # get the corresponding DS for each column - ds_list = grp_damage.columns.get_level_values(2).astype(int) - - # replace ones with the corresponding DS in each cell - grp_damage = grp_damage.mul(ds_list, axis=1) - - # aggregate across damage state indices - damage_groupby_2 = grp_damage.groupby(level=[0, 1], axis=1) - - # choose the max value - # i.e., the governing DS for each comp-loc pair - grp_damage = damage_groupby_2.max().mask( - damage_groupby_2.count() == 0, np.nan - ) - - # aggregate units to the same format - # assume identical units across locations for each comp - damage_units = damage_units.groupby(level=[0, 1], axis=1).first() - - else: - # otherwise, aggregate damage quantities for each comp - damage_groupby_2 = grp_damage.groupby(level=0, axis=1) - - # preserve NaNs - grp_damage = damage_groupby_2.sum().mask( - damage_groupby_2.count() == 0, np.nan - ) - - # and aggregate units to the same format - damage_units = damage_units.groupby(level=0, axis=1).first() - - if 'GroupedSample' in out_reqs: - grp_damage_s = pd.concat([grp_damage, damage_units]) - - grp_damage_s = convert_to_SimpleIndex(grp_damage_s, axis=1) - grp_damage_s.to_csv( - output_path / "DMG_grp.zip", - index_label=grp_damage_s.columns.name, - compression={ - 'method': 'zip', - 'archive_name': 'DMG_grp.csv', - }, - ) - out_files.append('DMG_grp.zip') - - if 'GroupedStatistics' in out_reqs: - grp_stats = describe(grp_damage) - grp_stats = pd.concat([grp_stats, damage_units]) - - grp_stats = convert_to_SimpleIndex(grp_stats, axis=1) - grp_stats.to_csv( - output_path / "DMG_grp_stats.csv", - index_label=grp_stats.columns.name, - ) - out_files.append('DMG_grp_stats.csv') - return damage_sample - - -def _asset_save(PAL, config, output_path, out_files): - cmp_sample, cmp_units = PAL.asset.save_cmp_sample(save_units=True) - cmp_units = cmp_units.to_frame().T - - if ( - get( - config, - 'DL/Outputs/Settings/AggregateColocatedComponentResults', - default=False, - ) - is True - ): - cmp_units = cmp_units.groupby(level=[0, 1, 2], axis=1).first() - - cmp_groupby_uid = cmp_sample.groupby(level=[0, 1, 2], axis=1) - - cmp_sample = cmp_groupby_uid.sum().mask(cmp_groupby_uid.count() == 0, np.nan) - - out_reqs = [ - out if val else "" for out, val in get(config, 'DL/Outputs/Asset').items() - ] - - if np.any(np.isin(['Sample', 'Statistics'], out_reqs)): - if 'Sample' in out_reqs: - cmp_sample_s = pd.concat([cmp_sample, cmp_units]) - - cmp_sample_s = convert_to_SimpleIndex(cmp_sample_s, axis=1) - cmp_sample_s.to_csv( - output_path / "CMP_sample.zip", - index_label=cmp_sample_s.columns.name, - compression={'method': 'zip', 'archive_name': 'CMP_sample.csv'}, - ) - out_files.append('CMP_sample.zip') - - if 'Statistics' in out_reqs: - cmp_stats = describe(cmp_sample) - cmp_stats = pd.concat([cmp_stats, cmp_units]) - - cmp_stats = convert_to_SimpleIndex(cmp_stats, axis=1) - cmp_stats.to_csv( - output_path / "CMP_stats.csv", index_label=cmp_stats.columns.name - ) - out_files.append('CMP_stats.csv') - - -def _demand_save(config, PAL, output_path, out_files): - out_reqs = [ - out if val else "" for out, val in get(config, 'DL/Outputs/Demand').items() - ] - - if np.any(np.isin(['Sample', 'Statistics'], out_reqs)): - demand_sample, demand_units = PAL.demand.save_sample(save_units=True) - - demand_units = demand_units.to_frame().T - - if 'Sample' in out_reqs: - demand_sample_s = pd.concat([demand_sample, demand_units]) - demand_sample_s = convert_to_SimpleIndex(demand_sample_s, axis=1) - demand_sample_s.to_csv( - output_path / "DEM_sample.zip", - index_label=demand_sample_s.columns.name, - compression={'method': 'zip', 'archive_name': 'DEM_sample.csv'}, - ) - out_files.append('DEM_sample.zip') - - if 'Statistics' in out_reqs: - demand_stats = describe(demand_sample) - demand_stats = pd.concat([demand_stats, demand_units]) - demand_stats = convert_to_SimpleIndex(demand_stats, axis=1) - demand_stats.to_csv( - output_path / "DEM_stats.csv", - index_label=demand_stats.columns.name, - ) - out_files.append('DEM_stats.csv') - return demand_sample - +def _result_summary( + assessment: DLCalculationAssessment, agg_repair: pd.DataFrame | None +) -> tuple[pd.DataFrame, pd.DataFrame]: + """ + Generate a summary of the results. -def _remove_existing_files(output_path): - # Initialize the array that we'll use to collect the output file names - out_files = [] + Parameters + ---------- + assessment : AssessmentBase + The assessment object. + agg_repair : pd.DataFrame + Aggregate repair data. - # Initialize the output folder - i.e., remove existing output files from - # there - files = os.listdir(output_path) - for filename in files: - if filename in out_files: - try: - os.remove(output_path / filename) - except OSError as exc: - raise OSError( - f'Error occurred while removing ' - f'`{output_path / filename}`: {exc}' - ) from exc - return out_files + Returns + ------- + tuple + Summary DataFrame and summary statistics DataFrame. + """ + damage_sample = assessment.damage.save_sample() + if damage_sample is None or agg_repair is None: + return pd.DataFrame(), pd.DataFrame() -def _summary(PAL, agg_repair, damage_sample, config, output_path, out_files): - damage_sample = damage_sample.groupby(level=[0, 3], axis=1).sum() + assert isinstance(damage_sample, pd.DataFrame) + damage_sample = damage_sample.groupby(level=['cmp', 'ds'], axis=1).sum() # type: ignore + assert isinstance(damage_sample, pd.DataFrame) damage_sample_s = convert_to_SimpleIndex(damage_sample, axis=1) if 'collapse-1' in damage_sample_s.columns: @@ -1147,1075 +1007,565 @@ def _summary(PAL, agg_repair, damage_sample, config, output_path, out_files): return summary, summary_stats -def _demand(config, config_path, length_unit, PAL, sample_size): - # check if there is a demand file location specified in the config file - if get(config, 'DL/Demands/DemandFilePath', default=False): - demand_path = Path(get(config, 'DL/Demands/DemandFilePath')).resolve() - - else: - # otherwise assume that there is a response.csv file next to the config file - demand_path = config_path.parent / 'response.csv' - - # try to load the demands - raw_demands = pd.read_csv(demand_path, index_col=0) - - # remove excessive demands that are considered collapses, if needed - if get(config, 'DL/Demands/CollapseLimits', default=False): - raw_demands = convert_to_MultiIndex(raw_demands, axis=1) - - if 'Units' in raw_demands.index: - raw_units = raw_demands.loc['Units', :] - raw_demands.drop('Units', axis=0, inplace=True) - - else: - raw_units = None - - DEM_to_drop = np.full(raw_demands.shape[0], False) - - for DEM_type, limit in get(config, 'DL/Demands/CollapseLimits').items(): - if raw_demands.columns.nlevels == 4: - DEM_to_drop += raw_demands.loc[:, idx[:, DEM_type, :, :]].max( - axis=1 - ) > float(limit) - - else: - DEM_to_drop += raw_demands.loc[:, idx[DEM_type, :, :]].max( - axis=1 - ) > float(limit) - - raw_demands = raw_demands.loc[~DEM_to_drop, :] - - if isinstance(raw_units, pd.Series): - raw_demands = pd.concat([raw_demands, raw_units.to_frame().T], axis=0) +def _parse_requested_output_file_names(output_config: dict) -> set[str]: + """ + Parse the output file names from the output configuration. - log_msg( - f"{np.sum(DEM_to_drop)} realizations removed from the demand " - f"input because they exceed the collapse limit. The remaining " - f"sample size: {raw_demands.shape[0]}" - ) + Parameters + ---------- + output_config : dict + Configuration for output files. - # add units to the demand data if needed - if "Units" not in raw_demands.index: - demands = add_units(raw_demands, length_unit) + Returns + ------- + set + Set of requested output file names. - else: - demands = raw_demands + """ + out_reqs = [] + for out, val in output_config.items(): + if val is True: + out_reqs.append(out) + return set(out_reqs) + + +def _demand_save( + output_config: dict, + assessment: DLCalculationAssessment, + output_path: Path, + out_files: list[str], +) -> None: + """ + Save demand results to files based on the output config. - # load the available demand sample - PAL.demand.load_sample(demands) + Parameters + ---------- + output_config : dict + Configuration for output files. + assessment : AssessmentBase + The assessment object. + output_path : Path + Path to the output directory. + out_files : list + List of output file names. - # get the calibration information - if get(config, 'DL/Demands/Calibration', default=False): - # then use it to calibrate the demand model - PAL.demand.calibrate_model(get(config, 'DL/Demands/Calibration')) + """ + out_reqs = _parse_requested_output_file_names(output_config) - else: - # if no calibration is requested, - # set all demands to use empirical distribution - PAL.demand.calibrate_model({"ALL": {"DistributionFamily": "empirical"}}) - - # and generate a new demand sample - PAL.demand.generate_sample( - { - "SampleSize": sample_size, - 'PreserveRawOrder': get( - config, 'DL/Demands/CoupledDemands', default=False - ), - 'DemandCloning': get(config, 'DL/Demands/DemandCloning', default=False), - } + demand_sample, demand_units_series = assessment.demand.save_sample( + save_units=True ) - - # get the generated demand sample - demand_sample, demand_units = PAL.demand.save_sample(save_units=True) - - demand_sample = pd.concat([demand_sample, demand_units.to_frame().T]) - - # get residual drift estimates, if needed - if get(config, 'DL/Demands/InferResidualDrift', default=False): - if get(config, 'DL/Demands/InferResidualDrift/method') == 'FEMA P-58': - RID_list = [] - PID = demand_sample['PID'].copy() - PID.drop('Units', inplace=True) - PID = PID.astype(float) - - for direction, delta_yield in get( - config, 'DL/Demands/InferResidualDrift' - ).items(): - if direction == 'method': - continue - - RID = PAL.demand.estimate_RID( - PID.loc[:, idx[:, direction]], - {'yield_drift': float(delta_yield)}, - ) - - RID_list.append(RID) - - RID = pd.concat(RID_list, axis=1) - RID_units = pd.Series( - ['unitless'] * RID.shape[1], - index=RID.columns, - name='Units', - ) - RID_sample = pd.concat([RID, RID_units.to_frame().T]) - demand_sample = pd.concat([demand_sample, RID_sample], axis=1) - - # add a constant one demand - demand_sample[('ONE', '0', '1')] = np.ones(demand_sample.shape[0]) - demand_sample.loc['Units', ('ONE', '0', '1')] = 'unitless' - - PAL.demand.load_sample(convert_to_SimpleIndex(demand_sample, axis=1)) - - -def _asset(config, PAL, demand_sample, cpref, csuff): - # set the number of stories - if get(config, 'DL/Asset/NumberOfStories', default=False): - PAL.stories = int(get(config, 'DL/Asset/NumberOfStories')) - - # load a component model and generate a sample - if get(config, 'DL/Asset/ComponentAssignmentFile', default=False): - cmp_marginals = pd.read_csv( - get(config, 'DL/Asset/ComponentAssignmentFile'), - index_col=0, - encoding_errors='replace', + assert isinstance(demand_sample, pd.DataFrame) + assert isinstance(demand_units_series, pd.Series) + demand_units = demand_units_series.to_frame().T + + if 'Sample' in out_reqs: + demand_sample_s = pd.concat([demand_sample, demand_units]) + demand_sample_s = convert_to_SimpleIndex(demand_sample_s, axis=1) + demand_sample_s.to_csv( + output_path / 'DEM_sample.zip', + index_label=demand_sample_s.columns.name, + compression={'method': 'zip', 'archive_name': 'DEM_sample.csv'}, ) + out_files.append('DEM_sample.zip') + + if 'Statistics' in out_reqs: + demand_stats = describe(demand_sample) + demand_stats = pd.concat([demand_stats, demand_units]) + demand_stats = convert_to_SimpleIndex(demand_stats, axis=1) + demand_stats.to_csv( + output_path / 'DEM_stats.csv', + index_label=demand_stats.columns.name, + ) + out_files.append('DEM_stats.csv') - DEM_types = demand_sample.columns.unique(level=0) - - # add component(s) to support collapse calculation - if get(config, 'DL/Damage/CollapseFragility', default=False): - coll_DEM = get(config, 'DL/Damage/CollapseFragility/DemandType') - if not coll_DEM.startswith('SA'): - # we need story-specific collapse assessment - # (otherwise we have a global demand and evaluate - # collapse directly, so this code should be skipped) - - if coll_DEM in DEM_types: - # excessive coll_DEM is added on every floor to detect large RIDs - cmp_marginals.loc['excessive.coll.DEM', 'Units'] = 'ea' - - locs = demand_sample[coll_DEM].columns.unique(level=0) - cmp_marginals.loc['excessive.coll.DEM', 'Location'] = ','.join( - locs - ) - - dirs = demand_sample[coll_DEM].columns.unique(level=1) - cmp_marginals.loc['excessive.coll.DEM', 'Direction'] = ','.join( - dirs - ) - - cmp_marginals.loc['excessive.coll.DEM', 'Theta_0'] = 1.0 - - else: - log_msg( - f'{cpref}WARNING: No {coll_DEM} among available ' - f'demands. Collapse ' - f'cannot be evaluated.{csuff}' - ) - - # always add a component to support basic collapse calculation - cmp_marginals.loc['collapse', 'Units'] = 'ea' - cmp_marginals.loc['collapse', 'Location'] = 0 - cmp_marginals.loc['collapse', 'Direction'] = 1 - cmp_marginals.loc['collapse', 'Theta_0'] = 1.0 - - # add components to support irreparable damage calculation - if not is_unspecified(config, 'DL/Damage/IrreparableDamage'): - if 'RID' in DEM_types: - # excessive RID is added on every floor to detect large RIDs - cmp_marginals.loc['excessiveRID', 'Units'] = 'ea' - - locs = demand_sample['RID'].columns.unique(level=0) - cmp_marginals.loc['excessiveRID', 'Location'] = ','.join(locs) - - dirs = demand_sample['RID'].columns.unique(level=1) - cmp_marginals.loc['excessiveRID', 'Direction'] = ','.join(dirs) - - cmp_marginals.loc['excessiveRID', 'Theta_0'] = 1.0 - - # irreparable is a global component to recognize is any of the - # excessive RIDs were triggered - cmp_marginals.loc['irreparable', 'Units'] = 'ea' - cmp_marginals.loc['irreparable', 'Location'] = 0 - cmp_marginals.loc['irreparable', 'Direction'] = 1 - cmp_marginals.loc['irreparable', 'Theta_0'] = 1.0 - - else: - log_msg( - f'{cpref}WARNING: No residual interstory drift ratio among' - f'available demands. Irreparable damage cannot be ' - f'evaluated.{csuff}' - ) - - # load component model - PAL.asset.load_cmp_model({'marginals': cmp_marginals}) - - # generate component quantity sample - PAL.asset.generate_cmp_sample() - - # if requested, load the quantity sample from a file - elif get(config, 'DL/Asset/ComponentSampleFile', default=False): - PAL.asset.load_cmp_sample(get(config, 'DL/Asset/ComponentSampleFile')) - return cmp_marginals - - -def _damage(config, custom_dl_file_path, PAL, cmp_marginals, length_unit): - if get(config, 'DL/Asset/ComponentDatabase') in default_DBs['fragility']: - component_db = [ - 'PelicunDefault/' - + default_DBs['fragility'][get(config, 'DL/Asset/ComponentDatabase')], - ] - else: - component_db = [] - - if get(config, 'DL/Asset/ComponentDatabasePath', default=False) is not False: - extra_comps = get(config, 'DL/Asset/ComponentDatabasePath') - - if 'CustomDLDataFolder' in extra_comps: - extra_comps = extra_comps.replace( - 'CustomDLDataFolder', custom_dl_file_path - ) - - component_db += [ - extra_comps, - ] - component_db = component_db[::-1] - - # prepare additional fragility data - - # get the database header from the default P58 db - P58_data = PAL.get_default_data('damage_DB_FEMA_P58_2nd') - - adf = pd.DataFrame(columns=P58_data.columns) - - if not is_unspecified(config, 'DL/Damage/CollapseFragility'): - if 'excessive.coll.DEM' in cmp_marginals.index: - # if there is story-specific evaluation - coll_CMP_name = 'excessive.coll.DEM' - else: - # otherwise, for global collapse evaluation - coll_CMP_name = 'collapse' - - adf.loc[coll_CMP_name, ('Demand', 'Directional')] = 1 - adf.loc[coll_CMP_name, ('Demand', 'Offset')] = 0 - - coll_DEM = get(config, 'DL/Damage/CollapseFragility/DemandType') - - if '_' in coll_DEM: - coll_DEM, coll_DEM_spec = coll_DEM.split('_') - else: - coll_DEM_spec = None - - coll_DEM_name = None - for demand_name, demand_short in EDP_to_demand_type.items(): - if demand_short == coll_DEM: - coll_DEM_name = demand_name - break - if coll_DEM_name is None: - raise ValueError('`coll_DEM_name` cannot be None.') +def _asset_save( + output_config: dict, + assessment: DLCalculationAssessment, + output_path: Path, + out_files: list[str], + *, + aggregate_colocated: bool = False, +) -> None: + """ + Save asset results to files based on the output config. - if coll_DEM_spec is None: - adf.loc[coll_CMP_name, ('Demand', 'Type')] = coll_DEM_name + Parameters + ---------- + output_config : dict + Configuration for output files. + assessment : AssessmentBase + The assessment object. + output_path : Path + Path to the output directory. + out_files : list + List of output file names. + aggregate_colocated : bool, optional + Whether to aggregate colocated components. Default is False. - else: - adf.loc[coll_CMP_name, ('Demand', 'Type')] = ( - f'{coll_DEM_name}|{coll_DEM_spec}' - ) + """ + output = assessment.asset.save_cmp_sample(save_units=True) + assert isinstance(output, tuple) + cmp_sample, cmp_units_series = output + cmp_units = cmp_units_series.to_frame().T + + if aggregate_colocated: + cmp_units = cmp_units.groupby(level=['cmp', 'loc', 'dir'], axis=1).first() # type: ignore + cmp_groupby_uid = cmp_sample.groupby(level=['cmp', 'loc', 'dir'], axis=1) # type: ignore + cmp_sample = cmp_groupby_uid.sum().mask(cmp_groupby_uid.count() == 0, np.nan) - coll_DEM_unit = add_units( - pd.DataFrame( - columns=[ - f'{coll_DEM}-1-1', - ] - ), - length_unit, - ).iloc[0, 0] + out_reqs = _parse_requested_output_file_names(output_config) - adf.loc[coll_CMP_name, ('Demand', 'Unit')] = coll_DEM_unit + if 'Sample' in out_reqs: + cmp_sample_s = pd.concat([cmp_sample, cmp_units]) - adf.loc[coll_CMP_name, ('LS1', 'Family')] = get( - config, - 'DL/Damage/CollapseFragility/CapacityDistribution', - default=np.nan, + cmp_sample_s = convert_to_SimpleIndex(cmp_sample_s, axis=1) + cmp_sample_s.to_csv( + output_path / 'CMP_sample.zip', + index_label=cmp_sample_s.columns.name, + compression={'method': 'zip', 'archive_name': 'CMP_sample.csv'}, ) + out_files.append('CMP_sample.zip') - adf.loc[coll_CMP_name, ('LS1', 'Theta_0')] = get( - config, 'DL/Damage/CollapseFragility/CapacityMedian', default=np.nan - ) + if 'Statistics' in out_reqs: + cmp_stats = describe(cmp_sample) + cmp_stats = pd.concat([cmp_stats, cmp_units]) - adf.loc[coll_CMP_name, ('LS1', 'Theta_1')] = get( - config, 'DL/Damage/CollapseFragility/Theta_1', default=np.nan + cmp_stats = convert_to_SimpleIndex(cmp_stats, axis=1) + cmp_stats.to_csv( + output_path / 'CMP_stats.csv', index_label=cmp_stats.columns.name ) + out_files.append('CMP_stats.csv') - adf.loc[coll_CMP_name, 'Incomplete'] = 0 - - if coll_CMP_name != 'collapse': - # for story-specific evaluation, we need to add a placeholder - # fragility that will never trigger, but helps us aggregate - # results in the end - adf.loc['collapse', ('Demand', 'Directional')] = 1 - adf.loc['collapse', ('Demand', 'Offset')] = 0 - adf.loc['collapse', ('Demand', 'Type')] = 'One' - adf.loc['collapse', ('Demand', 'Unit')] = 'unitless' - adf.loc['collapse', ('LS1', 'Theta_0')] = 1e10 - adf.loc['collapse', 'Incomplete'] = 0 - - elif is_unspecified(config, 'DL/Asset/ComponentDatabase/Water'): - # add a placeholder collapse fragility that will never trigger - # collapse, but allow damage processes to work with collapse - - adf.loc['collapse', ('Demand', 'Directional')] = 1 - adf.loc['collapse', ('Demand', 'Offset')] = 0 - adf.loc['collapse', ('Demand', 'Type')] = 'One' - adf.loc['collapse', ('Demand', 'Unit')] = 'unitless' - adf.loc['collapse', ('LS1', 'Theta_0')] = 1e10 - adf.loc['collapse', 'Incomplete'] = 0 - - if not is_unspecified(config, 'DL/Damage/IrreparableDamage'): - - # add excessive RID fragility according to settings provided in the - # input file - adf.loc['excessiveRID', ('Demand', 'Directional')] = 1 - adf.loc['excessiveRID', ('Demand', 'Offset')] = 0 - adf.loc['excessiveRID', ('Demand', 'Type')] = ( - 'Residual Interstory Drift Ratio' - ) - adf.loc['excessiveRID', ('Demand', 'Unit')] = 'unitless' - adf.loc['excessiveRID', ('LS1', 'Theta_0')] = get( - config, 'DL/Damage/IrreparableDamage/DriftCapacityMedian' - ) +def _damage_save( + output_config: dict, + assessment: DLCalculationAssessment, + output_path: Path, + out_files: list[str], + *, + aggregate_colocated: bool = False, + condense_ds: bool = False, +) -> None: + """ + Save damage results to files based on the output config. - adf.loc['excessiveRID', ('LS1', 'Family')] = "lognormal" + Parameters + ---------- + output_config : dict + Configuration for output files. + assessment : AssessmentBase + The assessment object. + output_path : Path + Path to the output directory. + out_files : list + List of output file names. + aggregate_colocated : bool, optional + Whether to aggregate colocated components. Default is False. + condense_ds : bool, optional + Whether to condense damage states. Default is False. - adf.loc['excessiveRID', ('LS1', 'Theta_1')] = get( - config, 'DL/Damage/IrreparableDamage/DriftCapacityLogStd' + """ + output = assessment.damage.save_sample(save_units=True) + assert isinstance(output, tuple) + damage_sample, damage_units_series = output + damage_units = damage_units_series.to_frame().T + + if aggregate_colocated: + damage_units = damage_units.groupby( # type: ignore + level=['cmp', 'loc', 'dir', 'ds'], axis=1 + ).first() + damage_groupby_uid = damage_sample.groupby( # type: ignore + level=['cmp', 'loc', 'dir', 'ds'], axis=1 ) - - adf.loc['excessiveRID', 'Incomplete'] = 0 - - # add a placeholder irreparable fragility that will never trigger - # damage, but allow damage processes to aggregate excessiveRID here - adf.loc['irreparable', ('Demand', 'Directional')] = 1 - adf.loc['irreparable', ('Demand', 'Offset')] = 0 - adf.loc['irreparable', ('Demand', 'Type')] = 'One' - adf.loc['irreparable', ('Demand', 'Unit')] = 'unitless' - adf.loc['irreparable', ('LS1', 'Theta_0')] = 1e10 - adf.loc['irreparable', 'Incomplete'] = 0 - - # TODO: we can improve this by creating a water - # network-specific assessment class - if not is_unspecified(config, 'DL/Asset/ComponentDatabase/Water'): - # add a placeholder aggregate fragility that will never trigger - # damage, but allow damage processes to aggregate the - # various pipeline damages - adf.loc['aggregate', ('Demand', 'Directional')] = 1 - adf.loc['aggregate', ('Demand', 'Offset')] = 0 - adf.loc['aggregate', ('Demand', 'Type')] = 'Peak Ground Velocity' - adf.loc['aggregate', ('Demand', 'Unit')] = 'mps' - adf.loc['aggregate', ('LS1', 'Theta_0')] = 1e10 - adf.loc['aggregate', ('LS2', 'Theta_0')] = 1e10 - adf.loc['aggregate', 'Incomplete'] = 0 - - PAL.damage.load_damage_model(component_db + [adf]) - - # load the damage process if needed - dmg_process = None - if get(config, 'DL/Damage/DamageProcess', default=False) is not False: - dp_approach = get(config, 'DL/Damage/DamageProcess') - - if dp_approach in damage_processes: - dmg_process = damage_processes[dp_approach] - - # For Hazus Earthquake, we need to specify the component ids - if dp_approach == 'Hazus Earthquake': - cmp_sample = PAL.asset.save_cmp_sample() - - cmp_list = cmp_sample.columns.unique(level=0) - - cmp_map = {'STR': '', 'LF': '', 'NSA': ''} - - for cmp in cmp_list: - for cmp_type in cmp_map: - if cmp_type + '.' in cmp: - cmp_map[cmp_type] = cmp - - new_dmg_process = dmg_process.copy() - for source_cmp, action in dmg_process.items(): - # first, look at the source component id - new_source = None - for cmp_type, cmp_id in cmp_map.items(): - if (cmp_type in source_cmp) and (cmp_id != ''): - new_source = source_cmp.replace(cmp_type, cmp_id) - break - - if new_source is not None: - new_dmg_process[new_source] = action - del new_dmg_process[source_cmp] - else: - new_source = source_cmp - - # then, look at the target component ids - for ds_i, target_vals in action.items(): - if isinstance(target_vals, str): - for cmp_type, cmp_id in cmp_map.items(): - if (cmp_type in target_vals) and (cmp_id != ''): - target_vals = target_vals.replace( - cmp_type, cmp_id - ) - - new_target_vals = target_vals - - else: - # we assume that target_vals is a list of str - new_target_vals = [] - - for target_val in target_vals: - for cmp_type, cmp_id in cmp_map.items(): - if (cmp_type in target_val) and (cmp_id != ''): - target_val = target_val.replace( - cmp_type, cmp_id - ) - - new_target_vals.append(target_val) - - new_dmg_process[new_source][ds_i] = new_target_vals - - dmg_process = new_dmg_process - - elif dp_approach == "User Defined": - # load the damage process from a file - with open( - get(config, 'DL/Damage/DamageProcessFilePath'), - 'r', - encoding='utf-8', - ) as f: - dmg_process = json.load(f) - - elif dp_approach == "None": - # no damage process applied for the calculation - dmg_process = None - - else: - log_msg(f"Prescribed Damage Process not recognized: " f"{dp_approach}") - - # calculate damages - PAL.damage.calculate(dmg_process=dmg_process) - - -def _loss(config, PAL, custom_dl_file_path, output_path, out_files): - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # backwards-compatibility for v3.2 and earlier | remove after v4.0 - if get(config, 'DL/Losses/BldgRepair', default=False): - update(config, 'DL/Losses/Repair', get(config, 'DL/Losses/BldgRepair')) - - if get(config, 'DL/Outputs/Loss/BldgRepair', default=False): - update( - config, - 'DL/Outputs/Loss/Repair', - get(config, 'DL/Outputs/Loss/BldgRepair'), + damage_sample = damage_groupby_uid.sum().mask( + damage_groupby_uid.count() == 0, np.nan ) - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # if requested, calculate repair consequences - if get(config, 'DL/Losses/Repair', default=False): + out_reqs = _parse_requested_output_file_names(output_config) - conseq_df, consequence_db = _load_consequence_info( - config, PAL, custom_dl_file_path - ) + if 'Sample' in out_reqs: + damage_sample_s = pd.concat([damage_sample, damage_units]) - # remove duplicates from conseq_df - conseq_df = conseq_df.loc[conseq_df.index.unique(), :] - - # add the replacement consequence to the data - adf = pd.DataFrame( - columns=conseq_df.columns, - index=pd.MultiIndex.from_tuples( - [ - ('replacement', 'Cost'), - ('replacement', 'Time'), - ('replacement', 'Carbon'), - ('replacement', 'Energy'), - ] - ), + damage_sample_s = convert_to_SimpleIndex(damage_sample_s, axis=1) + damage_sample_s.to_csv( + output_path / 'DMG_sample.zip', + index_label=damage_sample_s.columns.name, + compression={ + 'method': 'zip', + 'archive_name': 'DMG_sample.csv', + }, ) + out_files.append('DMG_sample.zip') - # DL_method = get(config, 'DL/Losses/Repair')['ConsequenceDatabase'] - DL_method = get(config, 'DL/Damage/DamageProcess', default='User Defined') - - _loss__cost(config, adf, DL_method) - - _loss__time(config, adf, DL_method, conseq_df) - - _loss__carbon(config, adf, DL_method) - - _loss__energy(config, adf, DL_method) + if 'Statistics' in out_reqs: + damage_stats = describe(damage_sample) + damage_stats = pd.concat([damage_stats, damage_units]) - # prepare the loss map - loss_map = None - if get(config, 'DL/Losses/Repair/MapApproach') == "Automatic": - # get the damage sample - loss_map = _loss__map_auto(PAL, conseq_df, DL_method, config) - - elif get(config, 'DL/Losses/Repair/MapApproach') == "User Defined": - loss_map = _loss__map_user(custom_dl_file_path, config) - - # prepare additional loss map entries, if needed - if 'DMG-collapse' not in loss_map.index: - loss_map.loc['DMG-collapse', 'Repair'] = 'replacement' - loss_map.loc['DMG-irreparable', 'Repair'] = 'replacement' - - # assemble the list of requested decision variables - DV_list = [] - if ( - get(config, 'DL/Losses/Repair/DecisionVariables', default=False) - is not False - ): - for DV_i, DV_status in get( - config, 'DL/Losses/Repair/DecisionVariables' - ).items(): - if DV_status is True: - DV_list.append(DV_i) - - else: - DV_list = None - - PAL.repair.load_model( - consequence_db + [adf], - loss_map, - decision_variables=DV_list, + damage_stats = convert_to_SimpleIndex(damage_stats, axis=1) + damage_stats.to_csv( + output_path / 'DMG_stats.csv', + index_label=damage_stats.columns.name, ) + out_files.append('DMG_stats.csv') - PAL.repair.calculate() - - agg_repair = PAL.repair.aggregate_losses() + if out_reqs.intersection({'GroupedSample', 'GroupedStatistics'}): + damage_groupby = damage_sample.groupby(level=['cmp', 'loc', 'ds'], axis=1) # type: ignore + damage_units = damage_units.groupby( + level=['cmp', 'loc', 'ds'], axis=1 + ).first() # type: ignore - # if requested, save results - if get(config, 'DL/Outputs/Loss/Repair', default=False): - _loss_save(PAL, config, output_path, out_files, agg_repair) + grp_damage = damage_groupby.sum().mask(damage_groupby.count() == 0, np.nan) - return agg_repair - return None + # if requested, condense DS output + if condense_ds: + # replace non-zero values with 1 + grp_damage = grp_damage.mask( + grp_damage.astype(np.float64).to_numpy() > 0, 1 + ) + # get the corresponding DS for each column + ds_list = grp_damage.columns.get_level_values('ds').astype(int) -def _load_consequence_info(config, PAL, custom_dl_file_path): - if get(config, 'DL/Losses/Repair/ConsequenceDatabase') in default_DBs['repair']: - consequence_db = [ - 'PelicunDefault/' - + default_DBs['repair'][ - get(config, 'DL/Losses/Repair/ConsequenceDatabase') - ], - ] + # replace ones with the corresponding DS in each cell + grp_damage = grp_damage.mul(ds_list, axis=1) - conseq_df = PAL.get_default_data( - default_DBs['repair'][ - get(config, 'DL/Losses/Repair/ConsequenceDatabase') - ][:-4] - ) - else: - consequence_db = [] + # aggregate across damage state indices + damage_groupby_2 = grp_damage.groupby(level=['cmp', 'loc'], axis=1) - conseq_df = pd.DataFrame() - - if ( - get(config, 'DL/Losses/Repair/ConsequenceDatabasePath', default=False) - is not False - ): - extra_comps = get(config, 'DL/Losses/Repair/ConsequenceDatabasePath') - - if 'CustomDLDataFolder' in extra_comps: - extra_comps = extra_comps.replace( - 'CustomDLDataFolder', custom_dl_file_path + # choose the max value + # i.e., the governing DS for each comp-loc pair + grp_damage = damage_groupby_2.max().mask( + damage_groupby_2.count() == 0, np.nan ) - consequence_db += [extra_comps] + # aggregate units to the same format + # assume identical units across locations for each comp + damage_units = damage_units.groupby(level=['cmp', 'loc'], axis=1).first() # type: ignore - extra_conseq_df = load_data( - extra_comps, - unit_conversion_factors=None, - orientation=1, - reindex=False, - ) - - if isinstance(conseq_df, pd.DataFrame): - conseq_df = pd.concat([conseq_df, extra_conseq_df]) else: - conseq_df = extra_conseq_df - - consequence_db = consequence_db[::-1] - return conseq_df, consequence_db + # otherwise, aggregate damage quantities for each comp + damage_groupby_2 = grp_damage.groupby(level='cmp', axis=1) + # preserve NaNs + grp_damage = damage_groupby_2.sum().mask( + damage_groupby_2.count() == 0, np.nan + ) -def _loss_save(PAL, config, output_path, out_files, agg_repair): - repair_sample, repair_units = PAL.repair.save_sample(save_units=True) - repair_units = repair_units.to_frame().T - - if ( - get( - config, - 'DL/Outputs/Settings/AggregateColocatedComponentResults', - default=False, - ) - is True - ): - repair_units = repair_units.groupby(level=[0, 1, 2, 3, 4, 5], axis=1).first() - - repair_groupby_uid = repair_sample.groupby(level=[0, 1, 2, 3, 4, 5], axis=1) + # and aggregate units to the same format + damage_units = damage_units.groupby(level='cmp', axis=1).first() # type: ignore - repair_sample = repair_groupby_uid.sum().mask( - repair_groupby_uid.count() == 0, np.nan - ) + if 'GroupedSample' in out_reqs: + grp_damage_s = pd.concat([grp_damage, damage_units]) - out_reqs = [ - out if val else "" - for out, val in get(config, 'DL/Outputs/Loss/Repair').items() - ] - - if np.any( - np.isin( - [ - 'Sample', - 'Statistics', - 'GroupedSample', - 'GroupedStatistics', - 'AggregateSample', - 'AggregateStatistics', - ], - out_reqs, - ) - ): - if 'Sample' in out_reqs: - repair_sample_s = repair_sample.copy() - repair_sample_s = pd.concat([repair_sample_s, repair_units]) - - repair_sample_s = convert_to_SimpleIndex(repair_sample_s, axis=1) - repair_sample_s.to_csv( - output_path / "DV_repair_sample.zip", - index_label=repair_sample_s.columns.name, + grp_damage_s = convert_to_SimpleIndex(grp_damage_s, axis=1) + grp_damage_s.to_csv( + output_path / 'DMG_grp.zip', + index_label=grp_damage_s.columns.name, compression={ 'method': 'zip', - 'archive_name': 'DV_repair_sample.csv', + 'archive_name': 'DMG_grp.csv', }, ) - out_files.append('DV_repair_sample.zip') + out_files.append('DMG_grp.zip') - if 'Statistics' in out_reqs: - repair_stats = describe(repair_sample) - repair_stats = pd.concat([repair_stats, repair_units]) + if 'GroupedStatistics' in out_reqs: + grp_stats = describe(grp_damage) + grp_stats = pd.concat([grp_stats, damage_units]) - repair_stats = convert_to_SimpleIndex(repair_stats, axis=1) - repair_stats.to_csv( - output_path / "DV_repair_stats.csv", - index_label=repair_stats.columns.name, + grp_stats = convert_to_SimpleIndex(grp_stats, axis=1) + grp_stats.to_csv( + output_path / 'DMG_grp_stats.csv', + index_label=grp_stats.columns.name, ) - out_files.append('DV_repair_stats.csv') - - if np.any(np.isin(['GroupedSample', 'GroupedStatistics'], out_reqs)): - repair_groupby = repair_sample.groupby(level=[0, 1, 2], axis=1) - - repair_units = repair_units.groupby(level=[0, 1, 2], axis=1).first() - - grp_repair = repair_groupby.sum().mask( - repair_groupby.count() == 0, np.nan - ) - - if 'GroupedSample' in out_reqs: - grp_repair_s = pd.concat([grp_repair, repair_units]) - - grp_repair_s = convert_to_SimpleIndex(grp_repair_s, axis=1) - grp_repair_s.to_csv( - output_path / "DV_repair_grp.zip", - index_label=grp_repair_s.columns.name, - compression={ - 'method': 'zip', - 'archive_name': 'DV_repair_grp.csv', - }, - ) - out_files.append('DV_repair_grp.zip') + out_files.append('DMG_grp_stats.csv') - if 'GroupedStatistics' in out_reqs: - grp_stats = describe(grp_repair) - grp_stats = pd.concat([grp_stats, repair_units]) - - grp_stats = convert_to_SimpleIndex(grp_stats, axis=1) - grp_stats.to_csv( - output_path / "DV_repair_grp_stats.csv", - index_label=grp_stats.columns.name, - ) - out_files.append('DV_repair_grp_stats.csv') - - if np.any(np.isin(['AggregateSample', 'AggregateStatistics'], out_reqs)): - if 'AggregateSample' in out_reqs: - agg_repair_s = convert_to_SimpleIndex(agg_repair, axis=1) - agg_repair_s.to_csv( - output_path / "DV_repair_agg.zip", - index_label=agg_repair_s.columns.name, - compression={ - 'method': 'zip', - 'archive_name': 'DV_repair_agg.csv', - }, - ) - out_files.append('DV_repair_agg.zip') - - if 'AggregateStatistics' in out_reqs: - agg_stats = convert_to_SimpleIndex(describe(agg_repair), axis=1) - agg_stats.to_csv( - output_path / "DV_repair_agg_stats.csv", - index_label=agg_stats.columns.name, - ) - out_files.append('DV_repair_agg_stats.csv') +def _loss_save( + output_config: dict, + assessment: DLCalculationAssessment, + output_path: Path, + out_files: list[str], + agg_repair: pd.DataFrame, + *, + aggregate_colocated: bool = False, +) -> None: + """ + Save loss results to files based on the output config. -def _loss__map_user(custom_dl_file_path, config): - if get(config, 'DL/Losses/Repair/MapFilePath', default=False) is not False: - loss_map_path = get(config, 'DL/Losses/Repair/MapFilePath') + Parameters + ---------- + output_config : dict + Configuration for output files. + assessment : AssessmentBase + The assessment object. + output_path : Path + Path to the output directory. + out_files : list + List of output file names. + agg_repair : pd.DataFrame + Aggregate repair data. + aggregate_colocated : bool, optional + Whether to aggregate colocated components. Default is False. - loss_map_path = loss_map_path.replace( - 'CustomDLDataFolder', custom_dl_file_path + """ + out = assessment.loss.ds_model.save_sample(save_units=True) + assert isinstance(out, tuple) + repair_sample, repair_units_series = out + repair_units = repair_units_series.to_frame().T + + if aggregate_colocated: + repair_units = repair_units.groupby( # type: ignore + level=['dv', 'loss', 'dmg', 'ds', 'loc', 'dir'], axis=1 + ).first() + repair_groupby_uid = repair_sample.groupby( # type: ignore + level=['dv', 'loss', 'dmg', 'ds', 'loc', 'dir'], axis=1 ) - - else: - raise ValueError('Missing loss map path.') - - loss_map = pd.read_csv(loss_map_path, index_col=0) - return loss_map - - -def _loss__map_auto(PAL, conseq_df, DL_method, config): - # get the damage sample - dmg_sample = PAL.damage.save_sample() - - # create a mapping for all components that are also in - # the prescribed consequence database - dmg_cmps = dmg_sample.columns.unique(level='cmp') - loss_cmps = conseq_df.index.unique(level=0) - - drivers = [] - loss_models = [] - - if DL_method in {'FEMA P-58', 'Hazus Hurricane'}: - # with these methods, we assume fragility and consequence data - # have the same IDs - - for dmg_cmp in dmg_cmps: - if dmg_cmp == 'collapse': - continue - - if dmg_cmp in loss_cmps: - drivers.append(f'DMG-{dmg_cmp}') - loss_models.append(dmg_cmp) - - elif DL_method in { - 'Hazus Earthquake', - 'Hazus Earthquake Transportation', - }: - # with Hazus Earthquake we assume that consequence - # archetypes are only differentiated by occupancy type - occ_type = get(config, 'DL/Asset/OccupancyType', default=None) - - for dmg_cmp in dmg_cmps: - if dmg_cmp == 'collapse': - continue - - cmp_class = dmg_cmp.split('.')[0] - if occ_type is not None: - loss_cmp = f'{cmp_class}.{occ_type}' - else: - loss_cmp = cmp_class - - if loss_cmp in loss_cmps: - drivers.append(f'DMG-{dmg_cmp}') - loss_models.append(loss_cmp) - - loss_map = pd.DataFrame(loss_models, columns=['Repair'], index=drivers) - return loss_map - - -def _loss__energy(config, adf, DL_method): - ren = ('replacement', 'Energy') - if 'ReplacementEnergy' in get(config, 'DL/Losses/Repair'): - ren = ('replacement', 'Energy') - - adf.loc[ren, ('Quantity', 'Unit')] = "1 EA" - - adf.loc[ren, ('DV', 'Unit')] = get( - config, 'DL/Losses/Repair/ReplacementEnergy/Unit' + repair_sample = repair_groupby_uid.sum().mask( + repair_groupby_uid.count() == 0, np.nan ) - adf.loc[ren, ('DS1', 'Theta_0')] = get( - config, 'DL/Losses/Repair/ReplacementEnergy/Median' - ) + out_reqs = _parse_requested_output_file_names(output_config) - if ( - pd.isna(get(config, 'DL/Losses/Repair/ReplacementEnergy/Distribution')) - is False - ): - adf.loc[ren, ('DS1', 'Family')] = get( - config, 'DL/Losses/Repair/ReplacementEnergy/Distribution' - ) - adf.loc[ren, ('DS1', 'Theta_1')] = get( - config, 'DL/Losses/Repair/ReplacementEnergy/Theta_1' - ) - else: - # add a default replacement energy value as a placeholder - # the default value depends on the consequence database - - # for FEMA P-58, use 0 kg - if DL_method == 'FEMA P-58': - adf.loc[ren, ('Quantity', 'Unit')] = '1 EA' - adf.loc[ren, ('DV', 'Unit')] = 'MJ' - adf.loc[ren, ('DS1', 'Theta_0')] = 0 - - else: - # for everything else, remove this consequence - adf.drop(ren, inplace=True) - - -def _loss__carbon(config, adf, DL_method): - rcarb = ('replacement', 'Carbon') - if not is_unspecified(config, 'DL/Losses/Repair/ReplacementCarbon'): - - rcarb = ('replacement', 'Carbon') - - adf.loc[rcarb, ('Quantity', 'Unit')] = "1 EA" - - adf.loc[rcarb, ('DV', 'Unit')] = get( - config, 'DL/Losses/Repair/ReplacementCarbon/Unit' - ) + if 'Sample' in out_reqs: + repair_sample_s = repair_sample.copy() + repair_sample_s = pd.concat([repair_sample_s, repair_units]) - adf.loc[rcarb, ('DS1', 'Theta_0')] = get( - config, 'DL/Losses/Repair/ReplacementCarbon/Median' + repair_sample_s = convert_to_SimpleIndex(repair_sample_s, axis=1) + repair_sample_s.to_csv( + output_path / 'DV_repair_sample.zip', + index_label=repair_sample_s.columns.name, + compression={ + 'method': 'zip', + 'archive_name': 'DV_repair_sample.csv', + }, ) + out_files.append('DV_repair_sample.zip') - if ( - pd.isna(get(config, 'DL/Losses/Repair/ReplacementCarbon/Distribution')) - is False - ): - adf.loc[rcarb, ('DS1', 'Family')] = get( - config, 'DL/Losses/Repair/ReplacementCarbon/Distribution' - ) - adf.loc[rcarb, ('DS1', 'Theta_1')] = get( - config, 'DL/Losses/Repair/ReplacementCarbon/Theta_1' - ) - else: - # add a default replacement carbon value as a placeholder - # the default value depends on the consequence database - - # for FEMA P-58, use 0 kg - if DL_method == 'FEMA P-58': - adf.loc[rcarb, ('Quantity', 'Unit')] = '1 EA' - adf.loc[rcarb, ('DV', 'Unit')] = 'kg' - adf.loc[rcarb, ('DS1', 'Theta_0')] = 0 - - else: - # for everything else, remove this consequence - adf.drop(rcarb, inplace=True) - - -def _loss__time(config, adf, DL_method, conseq_df): - rt = ('replacement', 'Time') - if not is_unspecified(config, 'DL/Losses/Repair/ReplacementTime'): - rt = ('replacement', 'Time') - - adf.loc[rt, ('Quantity', 'Unit')] = "1 EA" + if 'Statistics' in out_reqs: + repair_stats = describe(repair_sample) + repair_stats = pd.concat([repair_stats, repair_units]) - adf.loc[rt, ('DV', 'Unit')] = get( - config, 'DL/Losses/Repair/ReplacementTime/Unit' + repair_stats = convert_to_SimpleIndex(repair_stats, axis=1) + repair_stats.to_csv( + output_path / 'DV_repair_stats.csv', + index_label=repair_stats.columns.name, ) - - adf.loc[rt, ('DS1', 'Theta_0')] = get( - config, 'DL/Losses/Repair/ReplacementTime/Median' - ) - - if ( - pd.isna( - get( - config, - 'DL/Losses/Repair/ReplacementTime/Distribution', - default=np.nan, - ) - ) - is False - ): - adf.loc[rt, ('DS1', 'Family')] = get( - config, 'DL/Losses/Repair/ReplacementTime/Distribution' - ) - adf.loc[rt, ('DS1', 'Theta_1')] = get( - config, 'DL/Losses/Repair/ReplacementTime/Theta_1' + out_files.append('DV_repair_stats.csv') + + if out_reqs.intersection({'GroupedSample', 'GroupedStatistics'}): + repair_groupby = repair_sample.groupby(level=['dv', 'loss', 'dmg'], axis=1) # type: ignore + repair_units = repair_units.groupby( # type: ignore + level=['dv', 'loss', 'dmg'], axis=1 + ).first() + grp_repair = repair_groupby.sum().mask(repair_groupby.count() == 0, np.nan) + + if 'GroupedSample' in out_reqs: + grp_repair_s = pd.concat([grp_repair, repair_units]) + + grp_repair_s = convert_to_SimpleIndex(grp_repair_s, axis=1) + grp_repair_s.to_csv( + output_path / 'DV_repair_grp.zip', + index_label=grp_repair_s.columns.name, + compression={ + 'method': 'zip', + 'archive_name': 'DV_repair_grp.csv', + }, ) - else: - # add a default replacement time value as a placeholder - # the default value depends on the consequence database - - # for FEMA P-58, use 0 worker_days - if DL_method == 'FEMA P-58': - adf.loc[rt, ('Quantity', 'Unit')] = '1 EA' - adf.loc[rt, ('DV', 'Unit')] = 'worker_day' - adf.loc[rt, ('DS1', 'Theta_0')] = 0 - - # for Hazus EQ, use 1.0 as a loss_ratio - elif DL_method == 'Hazus Earthquake - Buildings': - adf.loc[rt, ('Quantity', 'Unit')] = '1 EA' - adf.loc[rt, ('DV', 'Unit')] = 'day' - - # load the replacement time that corresponds to total loss - occ_type = config['DL']['Asset']['OccupancyType'] - adf.loc[rt, ('DS1', 'Theta_0')] = conseq_df.loc[ - (f"STR.{occ_type}", 'Time'), ('DS5', 'Theta_0') - ] - - # otherwise, use 1 (and expect to have it defined by the user) - else: - adf.loc[rt, ('Quantity', 'Unit')] = '1 EA' - adf.loc[rt, ('DV', 'Unit')] = 'loss_ratio' - adf.loc[rt, ('DS1', 'Theta_0')] = 1 + out_files.append('DV_repair_grp.zip') + if 'GroupedStatistics' in out_reqs: + grp_stats = describe(grp_repair) + grp_stats = pd.concat([grp_stats, repair_units]) -def _loss__cost(config, adf, DL_method): - rc = ('replacement', 'Cost') - if not is_unspecified(config, 'DL/Losses/Repair/ReplacementCost'): - - adf.loc[rc, ('Quantity', 'Unit')] = "1 EA" - - adf.loc[rc, ('DV', 'Unit')] = get( - config, 'DL/Losses/Repair/ReplacementCost/Unit' - ) - - adf.loc[rc, ('DS1', 'Theta_0')] = get( - config, 'DL/Losses/Repair/ReplacementCost/Median' - ) - - if ( - pd.isna( - get( - config, - 'DL/Losses/Repair/ReplacementCost/Distribution', - default=np.nan, - ) + grp_stats = convert_to_SimpleIndex(grp_stats, axis=1) + grp_stats.to_csv( + output_path / 'DV_repair_grp_stats.csv', + index_label=grp_stats.columns.name, ) - is False - ): - adf.loc[rc, ('DS1', 'Family')] = get( - config, 'DL/Losses/Repair/ReplacementCost/Distribution' - ) - adf.loc[rc, ('DS1', 'Theta_1')] = get( - config, 'DL/Losses/Repair/ReplacementCost/Theta_1' + out_files.append('DV_repair_grp_stats.csv') + + if out_reqs.intersection({'AggregateSample', 'AggregateStatistics'}): + if 'AggregateSample' in out_reqs: + agg_repair_s = convert_to_SimpleIndex(agg_repair, axis=1) + agg_repair_s.to_csv( + output_path / 'DV_repair_agg.zip', + index_label=agg_repair_s.columns.name, + compression={ + 'method': 'zip', + 'archive_name': 'DV_repair_agg.csv', + }, ) + out_files.append('DV_repair_agg.zip') - else: - # add a default replacement cost value as a placeholder - # the default value depends on the consequence database - - # for FEMA P-58, use 0 USD - if DL_method == 'FEMA P-58': - adf.loc[rc, ('Quantity', 'Unit')] = '1 EA' - adf.loc[rc, ('DV', 'Unit')] = 'USD_2011' - adf.loc[rc, ('DS1', 'Theta_0')] = 0 - - # for Hazus EQ and HU, use 1.0 as a loss_ratio - elif DL_method in {'Hazus Earthquake', 'Hazus Hurricane'}: - adf.loc[rc, ('Quantity', 'Unit')] = '1 EA' - adf.loc[rc, ('DV', 'Unit')] = 'loss_ratio' + if 'AggregateStatistics' in out_reqs: + agg_stats = convert_to_SimpleIndex(describe(agg_repair), axis=1) + agg_stats.to_csv( + output_path / 'DV_repair_agg_stats.csv', + index_label=agg_stats.columns.name, + ) + out_files.append('DV_repair_agg_stats.csv') - # store the replacement cost that corresponds to total loss - adf.loc[rc, ('DS1', 'Theta_0')] = 1.00 - # otherwise, use 1 (and expect to have it defined by the user) - else: - adf.loc[rc, ('Quantity', 'Unit')] = '1 EA' - adf.loc[rc, ('DV', 'Unit')] = 'loss_ratio' - adf.loc[rc, ('DS1', 'Theta_0')] = 1 +def _remove_existing_files(output_path: Path, known_output_files: list[str]) -> None: + """ + Remove known existing files from the specified output path. + This function initializes the output folder by removing files that + already exist in the `known_output_files` list. -def _get_color_codes(color_warnings): - if color_warnings: - cpref = Fore.RED - csuff = Style.RESET_ALL - else: - cpref = csuff = '' - return cpref, csuff + Parameters + ---------- + output_path : Path + The path to the output folder where files are located. + known_output_files : list of str + A list of filenames that are expected to exist and should be + removed from the output folder. + Raises + ------ + OSError + If an error occurs while attempting to remove a file, an + OSError will be raised with the specific details of the + failure. -def main(): """ - Main method to parse arguments and run the pelicun calculation. + # Initialize the output folder - i.e., remove existing output files from + # there + files = os.listdir(output_path) + for filename in files: + if filename in known_output_files: + try: + (output_path / filename).unlink() + except OSError as exc: + msg = ( + f'Error occurred while removing ' + f'`{output_path / filename}`: {exc}' + ) + raise OSError(msg) from exc - """ - args = sys.argv[1:] + +def main() -> None: + """Parse arguments and run the pelicun calculation.""" + args_list = sys.argv[1:] parser = argparse.ArgumentParser() - parser.add_argument('-c', '--filenameDL') - parser.add_argument('-d', '--demandFile', default=None) - parser.add_argument('-s', '--Realizations', default=None) - parser.add_argument('--dirnameOutput', default=None) - parser.add_argument('--event_time', default=None) parser.add_argument( - '--detailed_results', default=True, type=str2bool, nargs='?', const=True + '-c', + '--filenameDL', + help='Path to the damage and loss (DL) configuration file.', ) parser.add_argument( - '--coupled_EDP', default=False, type=str2bool, nargs='?', const=False + '-d', + '--demandFile', + default=None, + help='Path to the file containing demand data.', ) parser.add_argument( - '--log_file', default=True, type=str2bool, nargs='?', const=True + '-s', + '--Realizations', + default=None, + help='Number of realizations to run in the probabilistic model.', ) parser.add_argument( - '--ground_failure', default=False, type=str2bool, nargs='?', const=False + '--dirnameOutput', + default=None, + help='Directory where output files will be stored.', ) - parser.add_argument('--auto_script', default=None) - parser.add_argument('--resource_dir', default=None) - parser.add_argument('--custom_model_dir', default=None) parser.add_argument( - '--regional', default=False, type=str2bool, nargs='?', const=False + '--detailed_results', + default=True, + type=str2bool, + nargs='?', + const=True, + help='Generate detailed results (True/False). Defaults to True.', ) - parser.add_argument('--output_format', default=None) parser.add_argument( - '--color_warnings', default=False, type=str2bool, nargs='?', const=False + '--coupled_EDP', + default=False, + type=str2bool, + nargs='?', + const=False, + help=( + 'Consider coupled Engineering Demand Parameters (EDPs) ' + 'in calculations (True/False). Defaults to False.' + ), ) - # parser.add_argument('-d', '--demandFile', default=None) - # parser.add_argument('--DL_Method', default = None) - # parser.add_argument('--outputBIM', default='BIM.csv') - # parser.add_argument('--outputEDP', default='EDP.csv') - # parser.add_argument('--outputDM', default='DM.csv') - # parser.add_argument('--outputDV', default='DV.csv') - - if not args: - print(f'Welcome. This is pelicun version {pelicun.__version__}') - print( - 'To access the documentation visit ' - 'https://nheri-simcenter.github.io/pelicun/index.html' - ) - print() + parser.add_argument( + '--log_file', + default=True, + type=str2bool, + nargs='?', + const=True, + help='Generate a log file (True/False). Defaults to True.', + ) + parser.add_argument( + '--auto_script', + default=None, + help='Optional path to a config auto-generation script.', + ) + parser.add_argument( + '--custom_model_dir', + default=None, + help='Directory containing custom model data.', + ) + parser.add_argument( + '--output_format', + default=None, + help='Desired output format for the results.', + ) + # TODO(JVM): fix color warnings + # parser.add_argument( + # '--color_warnings', + # default=False, + # type=str2bool, + # nargs='?', + # const=False, + # help=( + # 'Enable colored warnings in the console ' + # 'output (True/False). Defaults to False.' + # ), + # ) + parser.add_argument( + '--ground_failure', + default=False, + type=str2bool, + nargs='?', + const=False, + help='Currently not used. Soon to be deprecated.', + ) + parser.add_argument( + '--regional', + default=False, + type=str2bool, + nargs='?', + const=False, + help='Currently not used. Soon to be deprecated.', + ) + parser.add_argument('--resource_dir', default=None) + + if not args_list: parser.print_help() return - args = parser.parse_args(args) + args = parser.parse_args(args_list) - log_msg('Initializing pelicun calculation...') + log_msg('Initializing pelicun calculation.') - # print(args) - out = run_pelicun( - args.filenameDL, + run_pelicun( + config_path=args.filenameDL, demand_file=args.demandFile, output_path=args.dirnameOutput, realizations=args.Realizations, - detailed_results=args.detailed_results, - coupled_EDP=args.coupled_EDP, - log_file=args.log_file, - event_time=args.event_time, - ground_failure=args.ground_failure, auto_script_path=args.auto_script, - resource_dir=args.resource_dir, custom_model_dir=args.custom_model_dir, - color_warnings=args.color_warnings, - regional=args.regional, output_format=args.output_format, + detailed_results=args.detailed_results, + coupled_edp=args.coupled_EDP, ) - if out == -1: - log_msg("pelicun calculation failed.") - else: - log_msg('pelicun calculation completed.') + log_msg('pelicun calculation completed.') if __name__ == '__main__': diff --git a/pelicun/tools/HDF_to_CSV.py b/pelicun/tools/HDF_to_CSV.py index 21f8d1941..3cd27bddc 100644 --- a/pelicun/tools/HDF_to_CSV.py +++ b/pelicun/tools/HDF_to_CSV.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -33,28 +32,26 @@ # # You should have received a copy of the BSD 3-Clause License along with # pelicun. If not, see . -# -# Contributors: -# Adam Zsarnóczay from __future__ import annotations -import pandas as pd -import sys + import argparse +import sys from pathlib import Path +import pandas as pd -def convert_HDF(HDF_path): - HDF_ext = HDF_path.split('.')[-1] - CSV_base = HDF_path[: -len(HDF_ext) - 1] - HDF_path = Path(HDF_path).resolve() +def convert_HDF(hdf_path) -> None: # noqa: N802 + hdf_ext = hdf_path.split('.')[-1] + csv_base = hdf_path[: -len(hdf_ext) - 1] - store = pd.HDFStore(HDF_path) + hdf_path = Path(hdf_path).resolve() - for key in store.keys(): + store = pd.HDFStore(hdf_path) - store[key].to_csv(f'{CSV_base}_{key[1:].replace("/", "_")}.csv') + for key in store: + store[key].to_csv(f'{csv_base}_{key[1:].replace("/", "_")}.csv') store.close() diff --git a/pelicun/tools/__init__.py b/pelicun/tools/__init__.py new file mode 100644 index 000000000..cf08aa216 --- /dev/null +++ b/pelicun/tools/__init__.py @@ -0,0 +1,34 @@ +# noqa: D104 +# Copyright (c) 2018 Leland Stanford Junior University +# Copyright (c) 2018 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . diff --git a/pelicun/uq.py b/pelicun/uq.py index 3f7d59b5b..4a045689a 100644 --- a/pelicun/uq.py +++ b/pelicun/uq.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (c) 2018 Leland Stanford Junior University # Copyright (c) 2018 The Regents of the University of California @@ -33,47 +32,32 @@ # # You should have received a copy of the BSD 3-Clause License along with # pelicun. If not, see . -# -# Contributors: -# Adam Zsarnóczay -# John Vouvakis Manousakis - -""" -This module defines constants, classes and methods for uncertainty -quantification in pelicun. - -.. rubric:: Contents - -.. autosummary:: - - scale_distribution - mvn_orthotope_density - fit_distribution_to_sample - fit_distribution_to_percentiles - - RandomVariable - RandomVariableSet - RandomVariableRegistry - -""" +"""Constants, classes and methods for uncertainty quantification.""" from __future__ import annotations -from collections.abc import Callable + from abc import ABC, abstractmethod -from scipy.stats import uniform, norm # type: ignore -from scipy.stats import multivariate_normal as mvn # type: ignore -from scipy.stats import weibull_min -from scipy.stats._mvn import mvndst # type: ignore # pylint: disable=no-name-in-module # noqa # lol -from scipy.linalg import cholesky, svd # type: ignore -from scipy.optimize import minimize # type: ignore +from typing import TYPE_CHECKING + +import colorama import numpy as np import pandas as pd -import colorama -from colorama import Fore -from colorama import Style +from scipy.linalg import cholesky, svd # type: ignore +from scipy.optimize import minimize # type: ignore +from scipy.stats import multivariate_normal as mvn # type: ignore +from scipy.stats import norm, uniform, weibull_min # type: ignore +from scipy.stats._mvn import ( + mvndst, # type: ignore # noqa: PLC2701 +) + +if TYPE_CHECKING: + from collections.abc import Callable + + from pelicun.base import Logger colorama.init() +FIRST_POSITIVE_NUMBER = np.nextafter(0, 1) def scale_distribution( @@ -123,9 +107,9 @@ def scale_distribution( If the specified distribution family is unsupported. """ - if truncation_limits is not None: - truncation_limits = truncation_limits * scale_factor + truncation_limits = truncation_limits.copy() + truncation_limits *= scale_factor # undefined family is considered deterministic if pd.isna(family): @@ -148,14 +132,12 @@ def scale_distribution( theta_new[0] = theta[0] * scale_factor theta_new[1] = theta[1] * scale_factor - elif family == 'deterministic': - theta_new[0] = theta[0] * scale_factor - - elif family == 'multilinear_CDF': + elif family in {'deterministic', 'multilinear_CDF'}: theta_new[0] = theta[0] * scale_factor else: - raise ValueError(f'Unsupported distribution: {family}') + msg = f'Unsupported distribution: {family}' + raise ValueError(msg) return theta_new, truncation_limits @@ -202,7 +184,6 @@ def mvn_orthotope_density( Estimate of the error in the calculated probability density. """ - # process the inputs and get the number of dimensions mu = np.atleast_1d(mu) cov = np.atleast_2d(cov) @@ -243,10 +224,7 @@ def mvn_orthotope_density( np.putmask(infin, lowinf * uppinf, -1) # prepare the correlation coefficients - if ndim == 1: - correl = np.array([0.00]) - else: - correl = corr[np.tril_indices(ndim, -1)] + correl = np.array([0.0]) if ndim == 1 else corr[np.tril_indices(ndim, -1)] # estimate the density eps_alpha, alpha, _ = mvndst(lower, upper, infin, correl) @@ -258,7 +236,7 @@ def _get_theta( params: np.ndarray, inits: np.ndarray, dist_list: np.ndarray ) -> np.ndarray: """ - Returns the parameters of the target distributions. + Return the parameters of the target distributions. Uses the parameter values from the optimization algorithm (that are relative to the initial values) and the initial values to @@ -284,13 +262,10 @@ def _get_theta( If any of the distributions is unsupported. """ - theta = np.zeros(inits.shape) for i, (params_i, inits_i, dist_i) in enumerate(zip(params, inits, dist_list)): - if dist_i in {'normal', 'normal_std', 'lognormal'}: - # Standard deviation is used directly for 'normal' and # 'lognormal' sig = ( @@ -306,7 +281,6 @@ def _get_theta( theta[i, 1] = sig elif dist_i == 'normal_cov': - # Note that the CoV is used for 'normal_cov' sig = np.exp(np.log(inits_i[1]) + params_i[1]) @@ -317,7 +291,8 @@ def _get_theta( theta[i, 1] = sig else: - raise ValueError(f'Unsupported distribution: {dist_i}') + msg = f'Unsupported distribution: {dist_i}' + raise ValueError(msg) return theta @@ -348,24 +323,17 @@ def _get_limit_probs( If any of the distributions is unsupported. """ - if distribution in {'normal', 'normal_std', 'normal_cov', 'lognormal'}: a, b = limits mu = theta[0] sig = theta[1] if distribution != 'normal_COV' else np.abs(mu) * theta[1] - if np.isnan(a): - p_a = 0.0 - else: - p_a = norm.cdf((a - mu) / sig) - - if np.isnan(b): - p_b = 1.0 - else: - p_b = norm.cdf((b - mu) / sig) + p_a = 0.0 if np.isnan(a) else norm.cdf((a - mu) / sig) + p_b = 1.0 if np.isnan(b) else norm.cdf((b - mu) / sig) else: - raise ValueError(f'Unsupported distribution: {distribution}') + msg = f'Unsupported distribution: {distribution}' + raise ValueError(msg) return p_a, p_b @@ -405,7 +373,6 @@ def _get_std_samples( If any of the distributions is unsupported. """ - std_samples = np.zeros(samples.shape) for i, (samples_i, theta_i, tr_lim_i, dist_i) in enumerate( @@ -419,33 +386,37 @@ def _get_std_samples( True in (samples_i > lim_high).tolist() or True in (samples_i < lim_low).tolist() ): - raise ValueError( + msg = ( 'One or more sample values lie outside ' 'of the specified truncation limits.' ) + raise ValueError(msg) # first transform from normal to uniform - uni_samples = norm.cdf(samples_i, loc=theta_i[0], scale=theta_i[1]) + uni_sample = norm.cdf(samples_i, loc=theta_i[0], scale=theta_i[1]) # replace 0 and 1 values with the nearest float - uni_samples[uni_samples == 0] = np.nextafter(0, 1) - uni_samples[uni_samples == 1] = np.nextafter(1, -1) + uni_sample[uni_sample == 0] = np.nextafter(0, 1) + uni_sample[uni_sample == 1] = np.nextafter(1, -1) # consider truncation if needed p_a, p_b = _get_limit_probs(tr_lim_i, dist_i, theta_i) - uni_samples = (uni_samples - p_a) / (p_b - p_a) + uni_sample = (uni_sample - p_a) / (p_b - p_a) # then transform from uniform to standard normal - std_samples[i] = norm.ppf(uni_samples, loc=0.0, scale=1.0) + std_samples[i] = norm.ppf(uni_sample, loc=0.0, scale=1.0) else: - raise ValueError(f'Unsupported distribution: {dist_i}') + msg = f'Unsupported distribution: {dist_i}' + raise ValueError(msg) return std_samples def _get_std_corr_matrix(std_samples: np.ndarray) -> np.ndarray | None: """ + Estimate the correlation matrix. + Estimate the correlation matrix of the given standard normal samples. Ensure that the correlation matrix is positive semidefinite. @@ -467,9 +438,9 @@ def _get_std_corr_matrix(std_samples: np.ndarray) -> np.ndarray | None: If any of the elements of std_samples is np.inf or np.nan """ - if True in np.isinf(std_samples) or True in np.isnan(std_samples): - raise ValueError('std_samples array must not contain inf or NaN values') + msg = 'std_samples array must not contain inf or NaN values' + raise ValueError(msg) n_dims, n_samples = std_samples.shape @@ -492,7 +463,7 @@ def _get_std_corr_matrix(std_samples: np.ndarray) -> np.ndarray | None: # otherwise, we can try to fix the matrix using SVD except np.linalg.LinAlgError: try: - U, s, _ = svd( + u_matrix, s_vector, _ = svd( rho_hat, ) @@ -500,13 +471,15 @@ def _get_std_corr_matrix(std_samples: np.ndarray) -> np.ndarray | None: # if this also fails, we give up return None - S = np.diagflat(s) + s_diag = np.diagflat(s_vector) - rho_hat = U @ S @ U.T + rho_hat = u_matrix @ s_diag @ u_matrix.T np.fill_diagonal(rho_hat, 1.0) # check if we introduced any unreasonable values - if (np.max(rho_hat) > 1.01) or (np.min(rho_hat) < -1.01): + vmax = 1.01 + vmin = -1.01 + if (np.max(rho_hat) > vmax) or (np.min(rho_hat) < vmin): return None # round values to 1.0 and -1.0, if needed @@ -521,7 +494,7 @@ def _get_std_corr_matrix(std_samples: np.ndarray) -> np.ndarray | None: def _mvn_scale(x: np.ndarray, rho: np.ndarray) -> np.ndarray: """ - Scaling utility function + Scaling utility function. Parameters ---------- @@ -543,14 +516,14 @@ def _mvn_scale(x: np.ndarray, rho: np.ndarray) -> np.ndarray: rho_0 = np.eye(n_dims, n_dims) a = mvn.pdf(x, mean=np.zeros(n_dims), cov=rho_0) - a[a < 1.0e-10] = 1.0e-10 + a[a < FIRST_POSITIVE_NUMBER] = FIRST_POSITIVE_NUMBER b = mvn.pdf(x, mean=np.zeros(n_dims), cov=rho) return b / a -def _neg_log_likelihood( +def _neg_log_likelihood( # noqa: C901 params: np.ndarray, inits: np.ndarray, bnd_lower: np.ndarray, @@ -560,9 +533,11 @@ def _neg_log_likelihood( tr_limits: np.ndarray, det_limits: list[np.ndarray], censored_count: int, - enforce_bounds: bool = False, + enforce_bounds: bool = False, # noqa: FBT001, FBT002 ) -> float: """ + Calculate negative log likelihood. + Calculate the negative log likelihood of the given data samples given the parameter values and distribution information. @@ -572,18 +547,18 @@ def _neg_log_likelihood( Parameters ---------- - params : ndarray + params: ndarray 1D array with the parameter values to be assessed. - inits : ndarray + inits: ndarray 1D array with the initial estimates for the distribution parameters. - bnd_lower : ndarray + bnd_lower: ndarray 1D array with the lower bounds for the distribution parameters. - bnd_upper : ndarray + bnd_upper: ndarray 1D array with the upper bounds for the distribution parameters. - samples : ndarray + samples: ndarray 2D array with the data samples. Each column corresponds to a different random variable. dist_list: str ndarray of length D @@ -591,11 +566,11 @@ def _neg_log_likelihood( tr_limits: float ndarray Dx2 2D array with rows that represent [a, b] pairs of truncation limits. - det_limits : list + det_limits: list List with the detection limits for each random variable. - censored_count : int + censored_count: int Number of censored samples in the data. - enforce_bounds : bool, optional + enforce_bounds: bool, optional If True, the parameters are only considered valid if they are within the bounds defined by bnd_lower and bnd_upper. The default value is False. @@ -604,10 +579,11 @@ def _neg_log_likelihood( ------- float The negative log likelihood of the data given the distribution parameters. - """ + """ # First, check if the parameters are within the pre-defined bounds - # TODO: check if it is more efficient to use a bounded minimization algo + # TODO(AZ): check if it is more efficient to use a bounded + # minimization algo if enforce_bounds: if not ((params > bnd_lower) & (params < bnd_upper)).all(0): # if they are not, then return a large value to discourage the @@ -667,8 +643,8 @@ def _neg_log_likelihood( p_l, p_u = _get_limit_probs(det_lim_i, dist_i, theta_i) # rescale detection limits to consider truncation - p_l, p_u = [np.min([np.max([lim, p_a]), p_b]) for lim in (p_l, p_u)] - p_l, p_u = [(lim - p_a) / (p_b - p_a) for lim in (p_l, p_u)] + p_l, p_u = (np.min([np.max([lim, p_a]), p_b]) for lim in (p_l, p_u)) + p_l, p_u = ((lim - p_a) / (p_b - p_a) for lim in (p_l, p_u)) # transform limits to standard normal space det_lower[i], det_upper[i] = norm.ppf([p_l, p_u], loc=0.0, scale=1.0) @@ -696,8 +672,8 @@ def _neg_log_likelihood( # take the product of likelihoods calculated in each dimension scale = _mvn_scale(std_samples.T, rho_hat) - # TODO: We can almost surely replace the product of likelihoods with a call - # to mvn() + # TODO(AZ): We can almost surely replace the product of likelihoods + # with a call to mvn() likelihoods = np.prod(likelihoods, axis=0) * scale # Zeros are a result of limited floating point precision. Replace them @@ -706,27 +682,26 @@ def _neg_log_likelihood( likelihoods = np.clip(likelihoods, a_min=np.nextafter(0, 1), a_max=None) # calculate the total negative log likelihood - NLL = -( + negative_log_likelihood = -( np.sum(np.log(likelihoods)) # from samples + censored_count * np.log(cen_likelihood) ) # censoring influence - # normalize the NLL with the sample count - NLL = NLL / samples.size - # print(theta[0], params, NLL) - return NLL + # normalize the NLL with the sample count + return negative_log_likelihood / samples.size -def fit_distribution_to_sample( - raw_samples: np.ndarray, +def fit_distribution_to_sample( # noqa: C901 + raw_sample: np.ndarray, distribution: str | list[str], truncation_limits: tuple[float, float] = (np.nan, np.nan), censored_count: int = 0, detection_limits: tuple[float, float] = (np.nan, np.nan), + *, multi_fit: bool = False, - logger_object: None = None, + logger_object: Logger | None = None, ) -> tuple[np.ndarray, np.ndarray]: """ Fit a distribution to sample using maximum likelihood estimation. @@ -739,7 +714,7 @@ def fit_distribution_to_sample( Parameters ---------- - raw_samples: float ndarray + raw_sample: float ndarray Raw data that serves as the basis of estimation. The number of samples equals the number of columns and each row introduces a new feature. In other words: a list of sample lists is expected where each sample list @@ -748,7 +723,7 @@ def fit_distribution_to_sample( Defines the target probability distribution type. Different types of distributions can be mixed by providing a list rather than a single value. Each element of the list corresponds to one of the features in - the raw_samples. + the raw_sample. truncation_limits: float ndarray, optional, default: [None, None] Lower and/or upper truncation limits for the specified distributions. A two-element vector can be used for a univariate case, while two lists @@ -799,8 +774,7 @@ def fit_distribution_to_sample( If NaN values are produced during standard normal space transformation """ - - samples = np.atleast_2d(raw_samples) + samples = np.atleast_2d(raw_sample) tr_limits = np.atleast_2d(truncation_limits) det_limits = np.atleast_2d(detection_limits) dist_list = np.atleast_1d(distribution) @@ -866,10 +840,10 @@ def fit_distribution_to_sample( # There is nothing to gain from a time-consuming optimization if.. # the number of samples is too small - if (n_samples < 3) or ( + min_sample_size_for_optimization = 3 + if (n_samples < min_sample_size_for_optimization) or ( # there are no truncation or detection limits involved - np.all(np.isnan(tr_limits)) - and np.all(np.isnan(det_limits)) + np.all(np.isnan(tr_limits)) and np.all(np.isnan(det_limits)) ): # In this case, it is typically hard to improve on the method of # moments estimates for the parameters of the marginal distributions @@ -968,13 +942,14 @@ def fit_distribution_to_sample( # samples using that type of correlation (i.e., Gaussian copula) std_samples = _get_std_samples(samples, theta, tr_limits, dist_list) if True in np.isnan(std_samples) or True in np.isinf(std_samples): - raise ValueError( + msg = ( 'Something went wrong.' '\n' 'Conversion to standard normal space was unsuccessful. \n' 'The given samples might deviate ' 'substantially from the specified distribution.' ) + raise ValueError(msg) rho_hat = _get_std_corr_matrix(std_samples) if rho_hat is None: # If there is not enough data to produce a valid correlation matrix @@ -983,16 +958,16 @@ def fit_distribution_to_sample( np.fill_diagonal(rho_hat, 1.0) if logger_object: - logger_object.warn( - "Demand sample size too small to reliably estimate " - "the correlation matrix. Assuming uncorrelated demands." + logger_object.warning( + 'Demand sample size too small to reliably estimate ' + 'the correlation matrix. Assuming uncorrelated demands.' ) else: - print( - f"\n{Fore.RED}WARNING: Demand sample size " - f"too small to reliably estimate " - f"the correlation matrix. Assuming " - f"uncorrelated demands.{Style.RESET_ALL}" + print( # noqa: T201 + '\nWARNING: Demand sample size ' + 'too small to reliably estimate ' + 'the correlation matrix. Assuming ' + 'uncorrelated demands.' ) for d_i, distr in enumerate(dist_list): @@ -1006,15 +981,16 @@ def fit_distribution_to_sample( elif distr in {'normal', 'normal_cov'}: # replace standard deviation with coefficient of variation # note: this results in cov=inf if the mean is zero. - if np.abs(theta[d_i][0]) < 1.0e-40: + almost_zero = 1.0e-40 + if np.abs(theta[d_i][0]) < almost_zero: theta[d_i][1] = np.inf else: - theta[d_i][1] = theta[d_i][1] / np.abs(theta[d_i][0]) + theta[d_i][1] /= np.abs(theta[d_i][0]) return theta, rho_hat -def _OLS_percentiles( +def _OLS_percentiles( # noqa: N802 params: tuple[float, float], values: np.ndarray, perc: np.ndarray, family: str ) -> float: """ @@ -1022,13 +998,13 @@ def _OLS_percentiles( Parameters ---------- - params : tuple of floats + params: tuple of floats The parameters of the selected distribution family. - values : float ndarray + values: float ndarray The sample values for which the percentiles are requested. - perc : float ndarray + perc: float ndarray The requested percentile(s). - family : str + family: str The distribution family to use for the percentile estimation. Can be either 'normal' or 'lognormal'. @@ -1043,7 +1019,6 @@ def _OLS_percentiles( If `family` is not 'normal' or 'lognormal'. """ - if family == 'normal': theta_0 = params[0] theta_1 = params[1] @@ -1066,7 +1041,8 @@ def _OLS_percentiles( val_hat = np.exp(norm.ppf(perc, loc=np.log(theta_0), scale=theta_1)) else: - raise ValueError(f"Distribution family not recognized: {family}") + msg = f'Distribution family not recognized: {family}' + raise ValueError(msg) return np.sum((val_hat - values) ** 2.0) @@ -1098,7 +1074,6 @@ def fit_distribution_to_percentiles( Parameters of the fitted distribution. """ - out_list = [] percentiles_np = np.array(percentiles) @@ -1113,18 +1088,14 @@ def fit_distribution_to_percentiles( if family == 'normal': inits.append( - ( - np.abs(values[extreme_id] - inits[0]) - / np.abs(norm.ppf(percentiles_np[extreme_id], loc=0, scale=1)) - ) + np.abs(values[extreme_id] - inits[0]) + / np.abs(norm.ppf(percentiles_np[extreme_id], loc=0, scale=1)) ) elif family == 'lognormal': inits.append( - ( - np.abs(np.log(values[extreme_id] / inits[0])) - / np.abs(norm.ppf(percentiles_np[extreme_id], loc=0, scale=1)) - ) + np.abs(np.log(values[extreme_id] / inits[0])) + / np.abs(norm.ppf(percentiles_np[extreme_id], loc=0, scale=1)) ) out_list.append( @@ -1141,21 +1112,18 @@ def fit_distribution_to_percentiles( return families[best_out_id], out_list[best_out_id].x -class BaseRandomVariable(ABC): - """ - Base abstract class for different types of random variables. +class BaseRandomVariable(ABC): # noqa: B024 + """Base abstract class for different types of random variables.""" - """ - - __slots__ = [ - 'name', - 'distribution', - 'f_map', - '_uni_samples', + __slots__: list[str] = [ 'RV_set', - '_sample_DF', '_sample', + '_sample_DF', + '_uni_sample', 'anchor', + 'distribution', + 'f_map', + 'name', ] def __init__( @@ -1163,9 +1131,9 @@ def __init__( name: str, f_map: Callable | None = None, anchor: BaseRandomVariable | None = None, - ): + ) -> None: """ - Initializes a RandomVariable object. + Instantiate a RandomVariable object. Parameters ---------- @@ -1180,18 +1148,11 @@ def __init__( the attributes of this variable and its anchor do not have to be identical. - Raises - ------ - ValueError - If there are issues with the specified distribution theta - parameters. - """ - self.name = name self.distribution: str | None = None self.f_map = f_map - self._uni_samples: np.ndarray | None = None + self._uni_sample: np.ndarray | None = None self.RV_set: RandomVariableSet | None = None self._sample_DF: pd.Series | None = None self._sample: np.ndarray | None = None @@ -1230,7 +1191,7 @@ def sample(self, value: np.ndarray) -> None: self._sample_DF = pd.Series(value) @property - def sample_DF(self) -> pd.Series | None: + def sample_DF(self) -> pd.Series | None: # noqa: N802 """ Return the empirical or generated sample in a pandas Series. @@ -1257,12 +1218,14 @@ def uni_sample(self) -> np.ndarray | None: The sample from the controlling uniform distribution. """ - return self.anchor._uni_samples + if self.anchor is self: + return self._uni_sample + return self.anchor.uni_sample @uni_sample.setter def uni_sample(self, value: np.ndarray) -> None: """ - Assign the controlling sample to the random variable + Assign the controlling sample to the random variable. Parameters ---------- @@ -1270,36 +1233,39 @@ def uni_sample(self, value: np.ndarray) -> None: An array of floating point values in the [0, 1] domain. """ - self._uni_samples = value + self._uni_sample = value class RandomVariable(BaseRandomVariable): - """ - Random variable that needs `values` in `inverse_transform` - """ + """Random variable that needs `values` in `inverse_transform`.""" - __slots__: list[str] = [] + __slots__: list[str] = ['theta', 'truncation_limits'] - @abstractmethod def __init__( self, name: str, - theta: np.ndarray, + theta: np.ndarray | None, truncation_limits: np.ndarray | None = None, f_map: Callable | None = None, anchor: BaseRandomVariable | None = None, - ): + ) -> None: """ - Instantiates a normal random variable. + Instantiate a normal random variable. Parameters ---------- name: string A unique string that identifies the random variable. - theta: 2-element float ndarray + theta: float ndarray Set of parameters that define the Cumulative Distribution - Function (CDF) of the variable: Mean, coefficient of - variation. + Function (CDF) of the variable: E.g., mean and coefficient + of variation. Actual parameters depend on the distribution. + A 1D `theta` array represents constant parameters and results + in realizations that are all from the same distribution. + A 2D `theta` array represents variable parameters, meaning + that each realization will be sampled from the distribution + family that the object represents, but with the parameters + set for that realization. truncation_limits: float ndarray, optional Defines the np.array((a, b)) truncation limits for the distribution. Use np.nan to assign no limit in one direction, @@ -1316,15 +1282,177 @@ def __init__( """ if truncation_limits is None: truncation_limits = np.array((np.nan, np.nan)) + + # For backwards compatibility, cast to a numpy array if + # given a tuple or list. + if isinstance(theta, (list, tuple)): + theta = np.array(theta) + if isinstance(truncation_limits, (list, tuple)): + truncation_limits = np.array(truncation_limits) + + # Verify type + if theta is not None: + assert isinstance( + theta, np.ndarray + ), 'Parameter `theta` should be a numpy array.' + assert theta.ndim in { + 1, + 2, + }, 'Parameter `theta` can only be a 1D or 2D array.' + theta = np.atleast_1d(theta) + + assert isinstance( + truncation_limits, np.ndarray + ), 'Parameter `truncation_limits` should be a numpy array.' + assert truncation_limits.ndim in { + 1, + 2, + }, 'Parameter `truncation_limits` can only be a 1D or 2D array.' + # 1D corresponds to constant parameters. + # 2D corresponds to variable parameters (different in each + # realization). + + self.theta = theta + self.truncation_limits = truncation_limits + super().__init__( name=name, f_map=f_map, anchor=anchor, ) + def constant_parameters(self) -> bool: + """ + If the RV has constant or variable parameters. + + Constant parameters are the same in each realization. + + Returns + ------- + bool + True if the parameters are constant, false otherwise. + + """ + if self.theta is None: + return True + assert self.theta.ndim in {1, 2} + return self.theta.ndim == 1 + + def _prepare_theta_and_truncation_limit_arrays( + self, values: np.ndarray + ) -> tuple[np.ndarray, np.ndarray]: + """ + Prepare the `theta` and `truncation_limits` arrays. + + Prepare the `theta` and `truncation_limits` arrays for use in + calculations. This method adjusts the shape and size of the + `theta` and `truncation_limits` attributes to ensure + compatibility with the provided `values` array. The + adjustments enable support for two approaches: + * Constant parameters: The parameters remain the same + across all realizations. + * Variable parameters: The parameters vary across + realizations. + + Depending on whether the random variable uses constant or + variable parameters, the method ensures that the arrays are + correctly sized and broadcasted as needed. + + Parameters + ---------- + values : np.ndarray + Array of values for which the `theta` and + `truncation_limits` need to be prepared. The size of + `values` determines how the attributes are adjusted. + + Returns + ------- + tuple + A tuple containing: + * `theta` (np.ndarray): Adjusted array of parameters. + * `truncation_limits` (np.ndarray): Adjusted array of + truncation limits. + + Raises + ------ + ValueError + If the number of elements in `values` does not match the + number of rows of the `theta` attribute or if the + `truncation_limits` array is incompatible with the `theta` + array. + + Notes + ----- + The method ensures that `truncation_limits` are broadcasted to + match the shape of `theta` if needed. For constant parameters, + a single-row `theta` is expanded to a 2D array. For variable + parameters, the number of rows in `theta` must match the size + of `values`. + """ + theta = self.theta + assert theta is not None + truncation_limits = self.truncation_limits + assert truncation_limits is not None + if self.constant_parameters(): + theta = np.atleast_2d(theta) + assert theta is not None + elif len(values) != theta.shape[0]: + msg = ( + 'Number of elements in `values` variable should ' + 'match the number of rows of the parameter ' + 'attribute `theta`.' + ) + raise ValueError(msg) + + # Broadcast truncation limits to match shape + truncation_limits = np.atleast_2d(truncation_limits) + assert truncation_limits is not None + + if truncation_limits.shape != theta.shape: + # Number of rows should match + if truncation_limits.shape[1] != theta.shape[1]: + msg = 'Incompatible `truncation_limits` value.' + raise ValueError(msg) + truncation_limits = np.tile(truncation_limits, (theta.shape[0], 1)) + return theta, truncation_limits + + @staticmethod + def _ensure_positive_probability_difference( + p_b: np.ndarray, p_a: np.ndarray + ) -> None: + """ + Ensure there is probability mass between the truncation limits. + + Parameters + ---------- + p_b: float + The probability of not exceeding the upper truncation limit + based on the CDF of the random variable. + p_a: float + The probability of not exceeding the lower truncation limit + based on the CDF of the random variable. + + Raises + ------ + ValueError + If a negative probability difference is found. + + """ + if np.any((p_b - p_a) < FIRST_POSITIVE_NUMBER): + msg = ( + 'The probability mass within the truncation limits is ' + 'too small and the truncated distribution cannot be ' + 'sampled with sufficiently high accuracy. This is most ' + 'probably due to incorrect truncation limits set for ' + 'the distribution.' + ) + raise ValueError(msg) + @abstractmethod - def inverse_transform(self, values): + def inverse_transform(self, values: np.ndarray) -> np.ndarray: """ + Evaluate the inverse CDF. + Uses inverse probability integral transformation on the provided values. @@ -1332,23 +1460,22 @@ def inverse_transform(self, values): def inverse_transform_sampling(self) -> None: """ - Creates a sample using inverse probability integral - transformation. + Create a sample with inverse transform sampling. Raises ------ ValueError If there is no available uniform sample. + """ if self.uni_sample is None: - raise ValueError('No available uniform sample.') + msg = 'No available uniform sample.' + raise ValueError(msg) self.sample = self.inverse_transform(self.uni_sample) class UtilityRandomVariable(BaseRandomVariable): - """ - Random variable that needs `sample_size` in `inverse_transform` - """ + """Random variable that needs `sample_size` in `inverse_transform`.""" __slots__: list[str] = [] @@ -1358,9 +1485,9 @@ def __init__( name: str, f_map: Callable | None = None, anchor: BaseRandomVariable | None = None, - ): + ) -> None: """ - Instantiates a normal random variable. + Instantiate a normal random variable. Parameters ---------- @@ -1383,28 +1510,24 @@ def __init__( ) @abstractmethod - def inverse_transform(self, sample_size): + def inverse_transform(self, sample_size: int) -> np.ndarray: """ + Evaluate the inverse CDF. + Uses inverse probability integral transformation on the provided values. """ def inverse_transform_sampling(self, sample_size: int) -> None: - """ - Creates a sample using inverse probability integral - transformation. - """ + """Create a sample with inverse transform sampling.""" self.sample = self.inverse_transform(sample_size) class NormalRandomVariable(RandomVariable): - """ - Normal random variable. - - """ + """Normal random variable.""" - __slots__ = ['theta', 'truncation_limits'] + __slots__: list[str] = [] def __init__( self, @@ -1413,7 +1536,8 @@ def __init__( truncation_limits: np.ndarray | None = None, f_map: Callable | None = None, anchor: BaseRandomVariable | None = None, - ): + ) -> None: + """Instantiate a Normal random variable.""" if truncation_limits is None: truncation_limits = np.array((np.nan, np.nan)) super().__init__( @@ -1423,14 +1547,12 @@ def __init__( f_map=f_map, anchor=anchor, ) + assert self.theta is not None, '`theta` is required for Normal RVs' self.distribution = 'normal' - self.theta = np.atleast_1d(theta) - self.truncation_limits = truncation_limits def cdf(self, values: np.ndarray) -> np.ndarray: """ - Returns the Cumulative Density Function (CDF) at the specified - values. + Return the CDF at the given values. Parameters ---------- @@ -1443,17 +1565,20 @@ def cdf(self, values: np.ndarray) -> np.ndarray: 1D float ndarray containing CDF values """ - mu, sig = self.theta[:2] + theta, truncation_limits = self._prepare_theta_and_truncation_limit_arrays( + values + ) + mu, sig = theta.T if np.any(~np.isnan(self.truncation_limits)): - a, b = self.truncation_limits + a, b = truncation_limits.T - if np.isnan(a): - a = -np.inf - if np.isnan(b): - b = np.inf + # Replace NaN values + a = np.nan_to_num(a, nan=-np.inf) + b = np.nan_to_num(b, nan=np.inf) - p_a, p_b = [norm.cdf((lim - mu) / sig) for lim in (a, b)] + p_a, p_b = (norm.cdf((lim - mu) / sig) for lim in (a, b)) + self._ensure_positive_probability_difference(p_b, p_a) # cap the values at the truncation limits values = np.minimum(np.maximum(values, a), b) @@ -1469,8 +1594,10 @@ def cdf(self, values: np.ndarray) -> np.ndarray: return result - def inverse_transform(self, values): + def inverse_transform(self, values: np.ndarray) -> np.ndarray: """ + Evaluate the inverse CDF. + Evaluates the inverse of the Cumulative Density Function (CDF) for the given values. Used to generate random variable realizations. @@ -1485,34 +1612,21 @@ def inverse_transform(self, values): ndarray Inverse CDF values - Raises - ------ - ValueError - If the probability mass within the truncation limits is - too small - """ - - mu, sig = self.theta[:2] + theta, truncation_limits = self._prepare_theta_and_truncation_limit_arrays( + values + ) + mu, sig = theta.T if np.any(~np.isnan(self.truncation_limits)): - a, b = self.truncation_limits + a, b = truncation_limits.T - if np.isnan(a): - a = -np.inf - if np.isnan(b): - b = np.inf + # Replace NaN values + a = np.nan_to_num(a, nan=-np.inf) + b = np.nan_to_num(b, nan=np.inf) - p_a, p_b = [norm.cdf((lim - mu) / sig) for lim in (a, b)] - - if p_b - p_a == 0: - raise ValueError( - "The probability mass within the truncation limits is " - "too small and the truncated distribution cannot be " - "sampled with sufficiently high accuracy. This is most " - "probably due to incorrect truncation limits set for " - "the distribution." - ) + p_a, p_b = (norm.cdf((lim - mu) / sig) for lim in (a, b)) + self._ensure_positive_probability_difference(p_b, p_a) result = norm.ppf(values * (p_b - p_a) + p_a, loc=mu, scale=sig) @@ -1531,7 +1645,7 @@ class Normal_STD(NormalRandomVariable): """ - __slots__ = [] + __slots__: list[str] = [] def __init__( self, @@ -1540,7 +1654,8 @@ def __init__( truncation_limits: np.ndarray | None = None, f_map: Callable | None = None, anchor: BaseRandomVariable | None = None, - ): + ) -> None: + """Instantiate a Normal_STD random variable.""" mean, std = theta[:2] theta = np.array([mean, std]) super().__init__(name, theta, truncation_limits, f_map, anchor) @@ -1555,7 +1670,7 @@ class Normal_COV(NormalRandomVariable): """ - __slots__ = [] + __slots__: list[str] = [] def __init__( self, @@ -1564,11 +1679,22 @@ def __init__( truncation_limits: np.ndarray | None = None, f_map: Callable | None = None, anchor: BaseRandomVariable | None = None, - ): + ) -> None: + """ + Instantiate a Normal_COV random variable. + + Raises + ------ + ValueError + If the specified mean is zero. + + """ mean, cov = theta[:2] - if np.abs(mean) < 1e-40: - raise ValueError('The mean of Normal_COV RVs cannot be zero.') + almost_zero = 1e-40 + if np.abs(mean) < almost_zero: + msg = 'The mean of Normal_COV RVs cannot be zero.' + raise ValueError(msg) std = mean * cov theta = np.array([mean, std]) @@ -1576,21 +1702,19 @@ def __init__( class LogNormalRandomVariable(RandomVariable): - """ - Lognormal random variable. - - """ + """Lognormal random variable.""" - __slots__ = ['theta', 'truncation_limits'] + __slots__: list[str] = [] def __init__( self, name: str, theta: np.ndarray, - truncation_limits=None, - f_map=None, - anchor=None, - ): + truncation_limits: np.ndarray | None = None, + f_map: Callable | None = None, + anchor: BaseRandomVariable | None = None, + ) -> None: + """Instantiate a LogNormal random variable.""" if truncation_limits is None: truncation_limits = np.array((np.nan, np.nan)) super().__init__( @@ -1600,14 +1724,12 @@ def __init__( f_map=f_map, anchor=anchor, ) + assert self.theta is not None, '`theta` is required for LogNormal RVs' self.distribution = 'lognormal' - self.theta = np.atleast_1d(theta) - self.truncation_limits = truncation_limits def cdf(self, values: np.ndarray) -> np.ndarray: """ - Returns the Cumulative Density Function (CDF) at the specified - values. + Return the CDF at the given values. Parameters ---------- @@ -1617,22 +1739,25 @@ def cdf(self, values: np.ndarray) -> np.ndarray: Returns ------- ndarray - CDF values + 1D float ndarray containing CDF values """ - theta, beta = self.theta[:2] + theta, truncation_limits = self._prepare_theta_and_truncation_limit_arrays( + values + ) + theta, beta = theta.T if np.any(~np.isnan(self.truncation_limits)): - a, b = self.truncation_limits + a, b = truncation_limits.T - if np.isnan(a): - a = np.nextafter(0, 1) - if np.isnan(b): - b = np.inf + # Replace NaN values + a = np.nan_to_num(a, nan=np.nextafter(0, 1)) + b = np.nan_to_num(b, nan=np.inf) - p_a, p_b = [ + p_a, p_b = ( norm.cdf((np.log(lim) - np.log(theta)) / beta) for lim in (a, b) - ] + ) + self._ensure_positive_probability_difference(p_b, p_a) # cap the values at the truncation limits values = np.minimum(np.maximum(values, a), b) @@ -1652,9 +1777,10 @@ def cdf(self, values: np.ndarray) -> np.ndarray: def inverse_transform(self, values: np.ndarray) -> np.ndarray: """ - Evaluates the inverse of the Cumulative Density Function (CDF) - for the given values. Used to generate random variable - realizations. + Evaluate the inverse CDF. + + Uses inverse probability integral transformation on the + provided values. Parameters ---------- @@ -1667,23 +1793,23 @@ def inverse_transform(self, values: np.ndarray) -> np.ndarray: Inverse CDF values """ - - theta, beta = self.theta[:2] + theta, truncation_limits = self._prepare_theta_and_truncation_limit_arrays( + values + ) + theta, beta = theta.T if np.any(~np.isnan(self.truncation_limits)): - a, b = self.truncation_limits + a, b = truncation_limits.T - if np.isnan(a): - a = np.nextafter(0, 1) - else: - a = np.maximum(np.nextafter(0, 1), a) + # Replace NaN values + a = np.nan_to_num(a, nan=np.nextafter(0, 1)) + a[a <= 0] = np.nextafter(0, 1) + b = np.nan_to_num(b, nan=np.inf) - if np.isnan(b): - b = np.inf - - p_a, p_b = [ + p_a, p_b = ( norm.cdf((np.log(lim) - np.log(theta)) / beta) for lim in (a, b) - ] + ) + self._ensure_positive_probability_difference(p_b, p_a) result = np.exp( norm.ppf(values * (p_b - p_a) + p_a, loc=np.log(theta), scale=beta) @@ -1696,12 +1822,9 @@ def inverse_transform(self, values: np.ndarray) -> np.ndarray: class UniformRandomVariable(RandomVariable): - """ - Uniform random variable. - - """ + """Uniform random variable.""" - __slots__ = ['theta', 'truncation_limits'] + __slots__: list[str] = [] def __init__( self, @@ -1710,7 +1833,16 @@ def __init__( truncation_limits: np.ndarray | None = None, f_map: Callable | None = None, anchor: BaseRandomVariable | None = None, - ): + ) -> None: + """ + Instantiate a Uniform random variable. + + Raises + ------ + ValueError + If variable parameters are specified. + + """ if truncation_limits is None: truncation_limits = np.array((np.nan, np.nan)) super().__init__( @@ -1720,14 +1852,19 @@ def __init__( f_map=f_map, anchor=anchor, ) + assert self.theta is not None, '`theta` is required for Uniform RVs' self.distribution = 'uniform' - self.theta = np.atleast_1d(theta) - self.truncation_limits = truncation_limits + + if self.theta.ndim != 1: + msg = ( + 'Variable parameters are currently not supported for ' + 'Uniform random variables.' + ) + raise ValueError(msg) def cdf(self, values: np.ndarray) -> np.ndarray: """ - Returns the Cumulative Density Function (CDF) at the specified - values. + Return the CDF at the given values. Parameters ---------- @@ -1737,9 +1874,10 @@ def cdf(self, values: np.ndarray) -> np.ndarray: Returns ------- ndarray - CDF values + 1D float ndarray containing CDF values """ + assert self.theta is not None a, b = self.theta[:2] if np.isnan(a): @@ -1750,15 +1888,14 @@ def cdf(self, values: np.ndarray) -> np.ndarray: if np.any(~np.isnan(self.truncation_limits)): a, b = self.truncation_limits - result = uniform.cdf(values, loc=a, scale=(b - a)) - - return result + return uniform.cdf(values, loc=a, scale=(b - a)) def inverse_transform(self, values: np.ndarray) -> np.ndarray: """ - Evaluates the inverse of the Cumulative Density Function (CDF) - for the given values. Used to generate random variable - realizations. + Evaluate the inverse CDF. + + Uses inverse probability integral transformation on the + provided values. Parameters ---------- @@ -1771,6 +1908,7 @@ def inverse_transform(self, values: np.ndarray) -> np.ndarray: Inverse CDF values """ + assert self.theta is not None a, b = self.theta[:2] if np.isnan(a): @@ -1781,18 +1919,13 @@ def inverse_transform(self, values: np.ndarray) -> np.ndarray: if np.any(~np.isnan(self.truncation_limits)): a, b = self.truncation_limits - result = uniform.ppf(values, loc=a, scale=(b - a)) - - return result + return uniform.ppf(values, loc=a, scale=(b - a)) class WeibullRandomVariable(RandomVariable): - """ - Weibull random variable. - - """ + """Weibull random variable.""" - __slots__ = ['theta', 'truncation_limits'] + __slots__: list[str] = [] def __init__( self, @@ -1801,7 +1934,16 @@ def __init__( truncation_limits: np.ndarray | None = None, f_map: Callable | None = None, anchor: BaseRandomVariable | None = None, - ): + ) -> None: + """ + Instantiate a Weibull random variable. + + Raises + ------ + ValueError + If variable parameters are specified. + + """ if truncation_limits is None: truncation_limits = np.array((np.nan, np.nan)) super().__init__( @@ -1811,26 +1953,32 @@ def __init__( f_map=f_map, anchor=anchor, ) + assert self.theta is not None, '`theta` is required for Weibull RVs' self.distribution = 'weibull' - self.theta = np.atleast_1d(theta) - self.truncation_limits = truncation_limits + + if self.theta.ndim != 1: + msg = ( + 'Variable parameters are currently not supported for ' + 'Weibull random variables.' + ) + raise ValueError(msg) def cdf(self, values: np.ndarray) -> np.ndarray: """ - Returns the Cumulative Density Function (CDF) at the specified - values. + Return the CDF at the given values. Parameters ---------- values: 1D float ndarray - Values for which to evaluate the CDF. + Values for which to evaluate the CDF Returns ------- ndarray - CDF values. + 1D float ndarray containing CDF values """ + assert self.theta is not None lambda_, kappa = self.theta[:2] if np.any(~np.isnan(self.truncation_limits)): @@ -1842,7 +1990,8 @@ def cdf(self, values: np.ndarray) -> np.ndarray: if np.isnan(b): b = np.inf - p_a, p_b = [weibull_min.cdf(lim, kappa, scale=lambda_) for lim in (a, b)] + p_a, p_b = (weibull_min.cdf(lim, kappa, scale=lambda_) for lim in (a, b)) + self._ensure_positive_probability_difference(p_b, p_a) # cap the values at the truncation limits values = np.minimum(np.maximum(values, a), b) @@ -1864,22 +2013,23 @@ def cdf(self, values: np.ndarray) -> np.ndarray: def inverse_transform(self, values: np.ndarray) -> np.ndarray: """ - Evaluates the inverse of the Cumulative Density Function (CDF) - for the given values. Used to generate random variable - realizations. + Evaluate the inverse CDF. + + Uses inverse probability integral transformation on the + provided values. Parameters ---------- values: 1D float ndarray - Values for which to evaluate the inverse CDF. + Values for which to evaluate the inverse CDF Returns ------- ndarray - Inverse CDF values. + Inverse CDF values """ - + assert self.theta is not None lambda_, kappa = self.theta[:2] if np.any(~np.isnan(self.truncation_limits)): @@ -1893,7 +2043,8 @@ def inverse_transform(self, values: np.ndarray) -> np.ndarray: if np.isnan(b): b = np.inf - p_a, p_b = [weibull_min.cdf(lim, kappa, scale=lambda_) for lim in (a, b)] + p_a, p_b = (weibull_min.cdf(lim, kappa, scale=lambda_) for lim in (a, b)) + self._ensure_positive_probability_difference(p_b, p_a) result = weibull_min.ppf( values * (p_b - p_a) + p_a, kappa, scale=lambda_ @@ -1907,13 +2058,15 @@ def inverse_transform(self, values: np.ndarray) -> np.ndarray: class MultilinearCDFRandomVariable(RandomVariable): """ - Multilinear CDF random variable. This RV is defined by specifying - the points that define its Cumulative Density Function (CDF), and - linear interpolation between them. + Multilinear CDF random variable. + + This RV is defined by specifying the points that define its + Cumulative Density Function (CDF), and linear interpolation + between them. """ - __slots__ = ['theta'] + __slots__: list[str] = [] def __init__( self, @@ -1922,7 +2075,18 @@ def __init__( truncation_limits: np.ndarray | None = None, f_map: Callable | None = None, anchor: BaseRandomVariable | None = None, - ): + ) -> None: + """ + Instantiate a MultilinearCDF random variable. + + Raises + ------ + ValueError + In case of incompatible input parameters. + NotImplementedError + If truncation limits are specified. + + """ if truncation_limits is None: truncation_limits = np.array((np.nan, np.nan)) super().__init__( @@ -1932,55 +2096,57 @@ def __init__( f_map=f_map, anchor=anchor, ) + assert self.theta is not None, '`theta` is required for MultilinearCDF RVs' self.distribution = 'multilinear_CDF' if not np.all(np.isnan(truncation_limits)): - raise NotImplementedError( - f'{self.distribution} RVs do not support truncation' - ) + msg = f'{self.distribution} RVs do not support truncation' + raise NotImplementedError(msg) y_1 = theta[0, 1] if y_1 != 0.00: - raise ValueError( - "For multilinear CDF random variables, y_1 should be set to 0.00" - ) + msg = 'For multilinear CDF random variables, y_1 should be set to 0.00' + raise ValueError(msg) y_n = theta[-1, 1] if y_n != 1.00: - raise ValueError( - "For multilinear CDF random variables, y_n should be set to 1.00" - ) + msg = 'For multilinear CDF random variables, y_n should be set to 1.00' + raise ValueError(msg) x_s = theta[:, 0] if not np.array_equal(np.sort(x_s), x_s): - raise ValueError( - "For multilinear CDF random variables, " - "Xs should be specified in ascending order" + msg = ( + 'For multilinear CDF random variables, ' + 'Xs should be specified in ascending order' ) + raise ValueError(msg) if np.any(np.isclose(np.diff(x_s), 0.00)): - raise ValueError( - "For multilinear CDF random variables, " - "Xs should be specified in strictly ascending order" + msg = ( + 'For multilinear CDF random variables, ' + 'Xs should be specified in strictly ascending order' ) + raise ValueError(msg) y_s = theta[:, 1] if not np.array_equal(np.sort(y_s), y_s): - raise ValueError( - "For multilinear CDF random variables, " - "Ys should be specified in ascending order" + msg = ( + 'For multilinear CDF random variables, ' + 'Ys should be specified in ascending order' ) + raise ValueError(msg) if np.any(np.isclose(np.diff(y_s), 0.00)): - raise ValueError( - "For multilinear CDF random variables, " - "Ys should be specified in strictly ascending order" + msg = ( + 'For multilinear CDF random variables, ' + 'Ys should be specified in strictly ascending order' ) + raise ValueError(msg) - self.theta = np.atleast_1d(theta) + required_ndim = 2 + assert self.theta.ndim == required_ndim, 'Invalid `theta` dimensions.' def cdf(self, values: np.ndarray) -> np.ndarray: """ - Returns the Cumulative Density Function (CDF) at the specified - values. + Return the CDF at the given values. Parameters ---------- @@ -1990,22 +2156,22 @@ def cdf(self, values: np.ndarray) -> np.ndarray: Returns ------- ndarray - CDF values + 1D float ndarray containing CDF values """ + assert self.theta is not None x_i = [-np.inf] + [x[0] for x in self.theta] + [np.inf] y_i = [0.00] + [x[1] for x in self.theta] + [1.00] # Using Numpy's interp for linear interpolation - result = np.interp(values, x_i, y_i, left=0.00, right=1.00) - - return result + return np.interp(values, x_i, y_i, left=0.00, right=1.00) def inverse_transform(self, values: np.ndarray) -> np.ndarray: """ - Evaluates the inverse of the Cumulative Density Function (CDF) - for the given values. Used to generate random variable - realizations. + Evaluate the inverse CDF. + + Uses inverse probability integral transformation on the + provided values. Parameters ---------- @@ -2018,7 +2184,7 @@ def inverse_transform(self, values: np.ndarray) -> np.ndarray: Inverse CDF values """ - + assert self.theta is not None x_i = [x[0] for x in self.theta] y_i = [x[1] for x in self.theta] @@ -2029,46 +2195,43 @@ def inverse_transform(self, values: np.ndarray) -> np.ndarray: # extrapolate). # note: swapping the roles of x_i and y_i for inverse # interpolation - result = np.interp(values, y_i, x_i) - - return result + return np.interp(values, y_i, x_i) class EmpiricalRandomVariable(RandomVariable): - """ - Empirical random variable. + """Empirical random variable.""" - """ - - __slots__ = ['_raw_samples'] + __slots__: list[str] = ['_raw_sample'] def __init__( self, name: str, - raw_samples: np.ndarray, + theta: np.ndarray, truncation_limits: np.ndarray | None = None, f_map: Callable | None = None, anchor: BaseRandomVariable | None = None, - ): + ) -> None: + """Instantiate an Empirical random variable.""" if truncation_limits is None: truncation_limits = np.array((np.nan, np.nan)) super().__init__( name=name, - theta=raw_samples, + theta=None, truncation_limits=truncation_limits, f_map=f_map, anchor=anchor, ) self.distribution = 'empirical' if not np.all(np.isnan(truncation_limits)): - raise NotImplementedError( - f'{self.distribution} RVs do not support truncation' - ) + msg = f'{self.distribution} RVs do not support truncation' + raise NotImplementedError(msg) - self._raw_samples = np.atleast_1d(raw_samples) + self._raw_sample = np.atleast_1d(theta) def inverse_transform(self, values: np.ndarray) -> np.ndarray: """ + Evaluate the inverse CDF. + Maps given values to their corresponding positions within the empirical data array, simulating an inverse transformation based on the empirical distribution. This can be seen as a @@ -2088,35 +2251,31 @@ def inverse_transform(self, values: np.ndarray) -> np.ndarray: normalized positions. """ - s_ids = (values * len(self._raw_samples)).astype(int) - result = self._raw_samples[s_ids] - return result + s_ids = (values * len(self._raw_sample)).astype(int) + return self._raw_sample[s_ids] class CoupledEmpiricalRandomVariable(UtilityRandomVariable): - """ - Coupled empirical random variable. + """Coupled empirical random variable.""" - """ - - __slots__ = ['_raw_samples'] + __slots__: list[str] = ['_raw_sample'] def __init__( self, name: str, - raw_samples: np.ndarray, + theta: np.ndarray, truncation_limits: np.ndarray | None = None, f_map: Callable | None = None, anchor: BaseRandomVariable | None = None, - ): + ) -> None: """ - Instantiates a coupled empirical random variable. + Instantiate a coupled empirical random variable. Parameters ---------- name: string A unique string that identifies the random variable. - raw_samples: 1D float ndarray + theta: 1D float ndarray Samples from which to draw empirical realizations. truncation_limits: 2D float ndarray Not supported for CoupledEmpirical RVs. @@ -2145,14 +2304,15 @@ def __init__( ) self.distribution = 'coupled_empirical' if not np.all(np.isnan(truncation_limits)): - raise NotImplementedError( - f'{self.distribution} RVs do not support truncation' - ) + msg = f'{self.distribution} RVs do not support truncation' + raise NotImplementedError(msg) - self._raw_samples = np.atleast_1d(raw_samples) + self._raw_sample = np.atleast_1d(theta) def inverse_transform(self, sample_size: int) -> np.ndarray: """ + Evaluate the inverse CDF. + Generates a new sample array from the existing empirical data by repeating the dataset until it matches the requested sample size. @@ -2172,22 +2332,17 @@ def inverse_transform(self, sample_size: int) -> np.ndarray: dataset. """ - - raw_sample_count = len(self._raw_samples) + raw_sample_count = len(self._raw_sample) new_sample = np.tile( - self._raw_samples, int(sample_size / raw_sample_count) + 1 + self._raw_sample, int(sample_size / raw_sample_count) + 1 ) - result = new_sample[:sample_size] - return result + return new_sample[:sample_size] class DeterministicRandomVariable(UtilityRandomVariable): - """ - Deterministic random variable. - - """ + """Deterministic random variable.""" - __slots__ = ['theta'] + __slots__: list[str] = ['theta'] def __init__( self, @@ -2196,11 +2351,12 @@ def __init__( truncation_limits: np.ndarray | None = None, f_map: Callable | None = None, anchor: BaseRandomVariable | None = None, - ): + ) -> None: """ - Instantiates a deterministic random variable. This behaves - like a RandomVariable object but represents a specific, - deterministic value. + Instantiate a deterministic random variable. + + This behaves like a RandomVariable object but represents a + specific, deterministic value. Parameters ---------- @@ -2235,15 +2391,14 @@ def __init__( ) self.distribution = 'deterministic' if not np.all(np.isnan(truncation_limits)): - raise NotImplementedError( - f'{self.distribution} RVs do not support truncation' - ) + msg = f'{self.distribution} RVs do not support truncation' + raise NotImplementedError(msg) self.theta = np.atleast_1d(theta) def inverse_transform(self, sample_size: int) -> np.ndarray: """ - Generates samples that correspond to the value. + Evaluate the inverse CDF. Parameters ---------- @@ -2256,18 +2411,13 @@ def inverse_transform(self, sample_size: int) -> np.ndarray: Sample array containing the deterministic value. """ - - result = np.full(sample_size, self.theta[0]) - return result + return np.full(sample_size, self.theta[0]) class MultinomialRandomVariable(RandomVariable): - """ - Multinomial random variable. + """Multinomial random variable.""" - """ - - __slots__ = ['theta'] + __slots__: list[str] = [] def __init__( self, @@ -2276,7 +2426,18 @@ def __init__( truncation_limits: np.ndarray | None = None, f_map: Callable | None = None, anchor: BaseRandomVariable | None = None, - ): + ) -> None: + """ + Instantiate a Multinomial random variable. + + Raises + ------ + ValueError + In case of incompatible input parameters. + NotImplementedError + If truncation limits are specified. + + """ if truncation_limits is None: truncation_limits = np.array((np.nan, np.nan)) super().__init__( @@ -2287,22 +2448,23 @@ def __init__( anchor=anchor, ) if not np.all(np.isnan(truncation_limits)): - raise NotImplementedError( - f'{self.distribution} RVs do not support truncation' - ) + msg = f'{self.distribution} RVs do not support truncation' + raise NotImplementedError(msg) + assert self.theta is not None, '`theta` is required for Multinomial RVs' self.distribution = 'multinomial' if np.sum(theta) > 1.00: - raise ValueError( - f"The set of p values provided for a multinomial " - f"distribution shall sum up to less than or equal to 1.0. " - f"The provided values sum up to {np.sum(theta)}. p = " - f"{theta} ." + msg = ( + f'The set of p values provided for a multinomial ' + f'distribution shall sum up to less than or equal to 1.0. ' + f'The provided values sum up to {np.sum(theta)}. p = ' + f'{theta} .' ) - - self.theta = np.atleast_1d(theta) + raise ValueError(msg) def inverse_transform(self, values: np.ndarray) -> np.ndarray: """ + Evaluate the inverse CDF. + Transforms continuous values into discrete events based on the cumulative probabilities of the multinomial distribution derived by `theta`. @@ -2320,19 +2482,20 @@ def inverse_transform(self, values: np.ndarray) -> np.ndarray: Discrete events corresponding to the input values. """ + assert self.theta is not None p_cum = np.cumsum(self.theta)[:-1] for i, p_i in enumerate(p_cum): values[values < p_i] = 10 + i values[values <= 1.0] = 10 + len(p_cum) - result = values - 10 - - return result + return values - 10 class RandomVariableSet: """ + Random variable set. + Represents a set of random variables, each of which is described by its own probability distribution. The set allows the user to define correlations between the random variables, and provides @@ -2350,32 +2513,36 @@ class RandomVariableSet: Defines the correlation matrix that describes the correlation between the random variables in the set. Currently, only the Gaussian copula is supported. + """ - __slots__ = ['name', '_variables', '_Rho'] + __slots__: list[str] = ['_Rho', '_variables', 'name'] - def __init__(self, name: str, RV_list: list[RandomVariable], Rho: np.ndarray): + def __init__( + self, name: str, rv_list: list[BaseRandomVariable], rho: np.ndarray + ) -> None: + """Instantiate a random variable set.""" self.name = name - if len(RV_list) > 1: + if len(rv_list) > 1: # put the RVs in a dictionary for more efficient access - reorder = np.argsort([RV.name for RV in RV_list]) - self._variables = {RV_list[i].name: RV_list[i] for i in reorder} + reorder = np.argsort([RV.name for RV in rv_list]) + self._variables = {rv_list[i].name: rv_list[i] for i in reorder} # reorder the entries in the correlation matrix to correspond to the # sorted list of RVs - self._Rho = np.asarray(Rho[(reorder)].T[(reorder)].T) + self._Rho = np.asarray(rho[(reorder)].T[(reorder)].T) else: # if there is only one variable (for testing, probably) - self._variables = {rv.name: rv for rv in RV_list} - self._Rho = np.asarray(Rho) + self._variables = {rv.name: rv for rv in rv_list} + self._Rho = np.asarray(rho) # assign this RV_set to the variables - for _, var in self._variables.items(): + for var in self._variables.values(): var.RV_set = self @property - def RV(self) -> dict[str, RandomVariable]: + def RV(self) -> dict[str, RandomVariable]: # noqa: N802 """ Returns the random variable(s) assigned to the set. @@ -2413,9 +2580,9 @@ def sample(self) -> dict[str, np.ndarray | None]: """ return {name: rv.sample for name, rv in self._variables.items()} - def Rho(self, var_subset: list[str] | None = None) -> np.ndarray: + def Rho(self, var_subset: list[str] | None = None) -> np.ndarray: # noqa: N802 """ - Returns the (subset of the) correlation matrix. + Return the (subset of the) correlation matrix. Returns ------- @@ -2438,35 +2605,34 @@ def apply_correlation(self) -> None: correlations while preserving as much as possible from the correlation matrix. """ - - U_RV = np.array([RV.uni_sample for RV_name, RV in self.RV.items()]) + u_rv = np.array([RV.uni_sample for RV_name, RV in self.RV.items()]) # First try doing the Cholesky transformation try: - N_RV = norm.ppf(U_RV) + n_rv = norm.ppf(u_rv) - L = cholesky(self._Rho, lower=True) + l_mat = cholesky(self._Rho, lower=True) - NC_RV = L @ N_RV + nc_rv = l_mat @ n_rv - UC_RV = norm.cdf(NC_RV) + uc_rv = norm.cdf(nc_rv) except np.linalg.LinAlgError: # if the Cholesky doesn't work, we need to use the more # time-consuming but more robust approach based on SVD - N_RV = norm.ppf(U_RV) + n_rv = norm.ppf(u_rv) - U, s, _ = svd( + u_mat, s_mat, _ = svd( self._Rho, ) - S = np.diagflat(np.sqrt(s)) + s_diag = np.diagflat(np.sqrt(s_mat)) - NC_RV = (N_RV.T @ S @ U.T).T + nc_rv = (n_rv.T @ s_diag @ u_mat.T).T - UC_RV = norm.cdf(NC_RV) + uc_rv = norm.cdf(nc_rv) - for RV, uc_RV in zip(self.RV.values(), UC_RV): - RV.uni_sample = uc_RV + for rv, ucrv in zip(self.RV.values(), uc_rv): + rv.uni_sample = ucrv def orthotope_density( self, @@ -2509,7 +2675,6 @@ def orthotope_density( Estimate of the error in alpha. """ - if isinstance(lower, float): lower = np.array([lower]) if isinstance(upper, float): @@ -2545,7 +2710,7 @@ def orthotope_density( lower_std = lower_std.T upper_std = upper_std.T - OD = [ + od = [ mvn_orthotope_density( mu=np.zeros(len(variables)), cov=self.Rho(var_subset), @@ -2555,32 +2720,31 @@ def orthotope_density( for l_i, u_i in zip(lower_std, upper_std) ] - return np.asarray(OD) + return np.asarray(od) class RandomVariableRegistry: - """ - Description - - Parameters - ---------- + """Random variable registry.""" - """ + __slots__: list[str] = ['_rng', '_sets', '_variables'] - __slots__ = ['_rng', '_variables', '_sets'] - - def __init__(self, rng: np.random.Generator): + def __init__(self, rng: np.random.Generator) -> None: """ + Instantiate a random variable registry. + + Parameters + ---------- rng: numpy.random._generator.Generator Random variable generator object. - e.g.: np.random.default_rng(seed) + e.g.: np.random.default_rng(seed). + """ self._rng = rng - self._variables: dict[str, RandomVariable] = {} + self._variables: dict[str, BaseRandomVariable] = {} self._sets: dict[str, RandomVariableSet] = {} @property - def RV(self) -> dict[str, RandomVariable]: + def RV(self) -> dict[str, BaseRandomVariable]: # noqa: N802 """ Returns all random variable(s) in the registry. @@ -2592,9 +2756,9 @@ def RV(self) -> dict[str, RandomVariable]: """ return self._variables - def RVs(self, keys: list[str]) -> dict[str, RandomVariable]: + def RVs(self, keys: list[str]) -> dict[str, BaseRandomVariable]: # noqa: N802 """ - Returns a subset of the random variables in the registry + Return a subset of the random variables in the registry. Parameters ---------- @@ -2609,7 +2773,7 @@ def RVs(self, keys: list[str]) -> dict[str, RandomVariable]: """ return {name: self._variables[name] for name in keys} - def add_RV(self, RV: RandomVariable) -> None: + def add_RV(self, rv: BaseRandomVariable) -> None: # noqa: N802 """ Add a new random variable to the registry. @@ -2619,12 +2783,13 @@ def add_RV(self, RV: RandomVariable) -> None: When the RV already exists in the registry """ - if RV.name in self._variables: - raise ValueError(f'RV {RV.name} already exists in the registry.') - self._variables.update({RV.name: RV}) + if rv.name in self._variables: + msg = f'RV {rv.name} already exists in the registry.' + raise ValueError(msg) + self._variables.update({rv.name: rv}) @property - def RV_set(self) -> dict[str, RandomVariableSet]: + def RV_set(self) -> dict[str, RandomVariableSet]: # noqa: N802 """ Return the random variable set(s) in the registry. @@ -2636,14 +2801,12 @@ def RV_set(self) -> dict[str, RandomVariableSet]: """ return self._sets - def add_RV_set(self, RV_set: RandomVariableSet) -> None: - """ - Add a new set of random variables to the registry - """ - self._sets.update({RV_set.name: RV_set}) + def add_RV_set(self, rv_set: RandomVariableSet) -> None: # noqa: N802 + """Add a new set of random variables to the registry.""" + self._sets.update({rv_set.name: rv_set}) @property - def RV_sample(self) -> dict[str, np.ndarray | None]: + def RV_sample(self) -> dict[str, np.ndarray | None]: # noqa: N802 """ Return the sample for every random variable in the registry. @@ -2657,11 +2820,10 @@ def RV_sample(self) -> dict[str, np.ndarray | None]: def generate_sample(self, sample_size: int, method: str) -> None: """ - Generates samples for all variables in the registry. + Generate samples for all variables in the registry. Parameters ---------- - sample_size: int The number of samples requested per variable. method: str @@ -2678,10 +2840,9 @@ def generate_sample(self, sample_size: int, method: str) -> None: When the RV parent class is Unknown """ - # Generate a dictionary with IDs of the free (non-anchored and # non-deterministic) variables - RV_list = [ + rv_list = [ RV_name for RV_name, RV in self.RV.items() if ( @@ -2689,70 +2850,74 @@ def generate_sample(self, sample_size: int, method: str) -> None: or (RV.distribution in {'deterministic', 'coupled_empirical'}) ) ] - RV_ID = {RV_name: ID for ID, RV_name in enumerate(RV_list)} - RV_count = len(RV_ID) + rv_id = {RV_name: ID for ID, RV_name in enumerate(rv_list)} + rv_count = len(rv_id) # Generate controlling samples from a uniform distribution for free RVs if 'LHS' in method: bin_low = np.array( - [self._rng.permutation(sample_size) for i in range(RV_count)] + [self._rng.permutation(sample_size) for i in range(rv_count)] ) if method == 'LHS_midpoint': - U_RV = np.ones([RV_count, sample_size]) * 0.5 - U_RV = (bin_low + U_RV) / sample_size + u_rv = np.ones([rv_count, sample_size]) * 0.5 + u_rv = (bin_low + u_rv) / sample_size elif method == 'LHS': - U_RV = self._rng.random(size=[RV_count, sample_size]) - U_RV = (bin_low + U_RV) / sample_size + u_rv = self._rng.random(size=[rv_count, sample_size]) + u_rv = (bin_low + u_rv) / sample_size elif method == 'MonteCarlo': - U_RV = self._rng.random(size=[RV_count, sample_size]) + u_rv = self._rng.random(size=[rv_count, sample_size]) # Assign the controlling samples to the RVs - for RV_name, RV_id in RV_ID.items(): - self.RV[RV_name].uni_sample = U_RV[RV_id] + for rv_name, rvid in rv_id.items(): + self.RV[rv_name].uni_sample = u_rv[rvid] # Apply correlations for the pre-defined RV sets - for RV_set in self.RV_set.values(): + for rv_set in self.RV_set.values(): # prepare the correlated uniform distribution for the set - RV_set.apply_correlation() + rv_set.apply_correlation() # Convert from uniform to the target distribution for every RV - for RV in self.RV.values(): - if isinstance(RV, UtilityRandomVariable): - RV.inverse_transform_sampling(sample_size) - elif isinstance(RV, RandomVariable): - RV.inverse_transform_sampling() + for rv in self.RV.values(): + if isinstance(rv, UtilityRandomVariable): + rv.inverse_transform_sampling(sample_size) + elif isinstance(rv, RandomVariable): + rv.inverse_transform_sampling() else: - raise NotImplementedError('Unknown RV parent class.') + msg = 'Unknown RV parent class.' + raise NotImplementedError(msg) -def rv_class_map(distribution_name: str) -> type[BaseRandomVariable]: +def rv_class_map( + distribution_name: str, +) -> type[RandomVariable | UtilityRandomVariable]: """ - Maps convenient distribution names to their corresponding random - variable class. + Map convenient distributions to their corresponding class. Parameters ---------- distribution_name: str - The name of a distribution. + The name of a distribution. Returns ------- - RandomVariable - RandomVariable class. + type[RandomVariable | UtilityRandomVariable] + The class of the corresponding random variable. Raises ------ ValueError - If the given distribution name does not correspond to a - distribution class. + If the given distribution name does not correspond to a + distribution class. """ if pd.isna(distribution_name): distribution_name = 'deterministic' - distribution_map = { + + # Mapping for RandomVariable subclasses + random_variable_map: dict[str, type[RandomVariable]] = { 'normal': Normal_COV, 'normal_std': Normal_STD, 'normal_cov': Normal_COV, @@ -2761,10 +2926,18 @@ def rv_class_map(distribution_name: str) -> type[BaseRandomVariable]: 'weibull': WeibullRandomVariable, 'multilinear_CDF': MultilinearCDFRandomVariable, 'empirical': EmpiricalRandomVariable, + 'multinomial': MultinomialRandomVariable, + } + + # Mapping for UtilityRandomVariable subclasses + utility_random_variable_map: dict[str, type[UtilityRandomVariable]] = { 'coupled_empirical': CoupledEmpiricalRandomVariable, 'deterministic': DeterministicRandomVariable, - 'multinomial': MultinomialRandomVariable, } - if distribution_name not in distribution_map: - raise ValueError(f'Unsupported distribution: {distribution_name}') - return distribution_map[distribution_name] + + if distribution_name in random_variable_map: + return random_variable_map[distribution_name] + if distribution_name in utility_random_variable_map: + return utility_random_variable_map[distribution_name] + msg = f'Unsupported distribution: {distribution_name}' + raise ValueError(msg) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..376384543 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,45 @@ +[tool.ruff] +line-length = 85 +exclude = [ + "rulesets", + "pelicun/tests/dl_calculation/e7/auto_HU_NJ.py", + "pelicun/tests/dl_calculation/e8/auto_HU_LA.py", + "pelicun/tests/dl_calculation/e9/custom_pop.py" +] + +[tool.ruff.lint] +# Enable all known categories +select = ["ALL"] +ignore = ["ANN101", "D211", "D212", "Q000", "Q003", "COM812", "D203", "ISC001", "E501", "ERA001", "PGH003", "FIX002", "TD003", "S101", "N801", "S311", "G004", "SIM102", "SIM108", "NPY002", "F401"] +preview = true + +[tool.ruff.lint.pydocstyle] +convention = "numpy" + +[tool.ruff.lint.pylint] +max-args=15 +max-locals=50 +max-returns=11 +max-branches=50 +max-statements=150 +max-bool-expr=5 + +[tool.ruff.lint.per-file-ignores] +"pelicun/tests/*" = ["D", "N802", "SLF001", "PLR2004", "PLR6301"] +"pelicun/resources/auto/*" = ['PLR', 'T', 'N', 'ANN', 'D', 'PTH', 'INP', 'DOC', 'RET', 'TD'] +"pelicun/tools/HDF_to_CSV.py" = ["ALL"] +"pelicun/tests/validation/inactive/*" = ["T201", "B018", "ANN", "PD"] +"pelicun/tests/dl_calculation/rulesets/*" = ["N999"] +"doc/source/examples/notebooks/*" = ["INP001", "CPY001", "D400", "B018", "F821", "T201", "T203", "F404", "E402"] + +[tool.ruff.format] +quote-style = "single" + +[tool.codespell] +ignore-words = ["ignore_words.txt"] +skip = ["*.html", "./htmlcov/*", "./doc_src/build/*", "./pelicun.egg-info/*", "./doc_src/*", "./doc/build/*", "*/rulesets/*", "custom_pop.py", "*/SimCenterDBDL/*", "auto_HU_NJ.py", "auto_HU_LA.py", "custom_pop.py"] + +[tool.mypy] +ignore_missing_imports = true +exclude = "flycheck" +namespace_packages = false diff --git a/pytest.ini b/pytest.ini index 7640f4f7f..2762a03a5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,7 @@ [pytest] filterwarnings = ignore:.*errors='ignore' is deprecated and will raise.*:FutureWarning + ignore:.*Downcasting object dtype arrays on.*:FutureWarning ignore:.*invalid value encountered in multiply.*:RuntimeWarning ignore:.*invalid value encountered in add.*:RuntimeWarning ignore:.*DataFrameGroupBy\.apply operated on the grouping columns.*:DeprecationWarning diff --git a/run_checks.sh b/run_checks.sh new file mode 100755 index 000000000..7b5487bec --- /dev/null +++ b/run_checks.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Spell-check +echo "Spell-checking." +echo +codespell . +if [ $? -ne 0 ]; then + echo "Spell-checking failed." + exit 1 +fi + +# Check formatting with ruff +echo "Checking formatting with 'ruff format --diff'." +echo +ruff format --diff +if [ $? -ne 0 ]; then + echo "ruff format failed." + exit 1 +fi + +# Run ruff for linting +echo "Linting with 'ruff check --fix'." +echo +ruff check --fix --output-format concise +if [ $? -ne 0 ]; then + echo "ruff check failed." + exit 1 +fi + +# Run mypy for type checking +echo "Type checking with mypy." +echo +mypy pelicun +if [ $? -ne 0 ]; then + echo "mypy failed. Exiting." + exit 1 +fi + +# Run pytest for testing and generate coverage report +echo "Running unit-tests." +echo +python -m pytest pelicun/tests --cov=pelicun --cov-report html -n auto +if [ $? -ne 0 ]; then + echo "pytest failed. Exiting." + exit 1 +fi + +echo "All checks passed successfully." +echo diff --git a/setup.py b/setup.py index 06616dc9d..b45fde937 100644 --- a/setup.py +++ b/setup.py @@ -1,28 +1,68 @@ -""" -setup.py file of the `pelicun` package. +# +# Copyright (c) 2023 Leland Stanford Junior University +# Copyright (c) 2023 The Regents of the University of California +# +# This file is part of pelicun. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# You should have received a copy of the BSD 3-Clause License along with +# pelicun. If not, see . -""" +"""setup.py file of the `pelicun` package.""" + +from pathlib import Path + +from setuptools import find_packages, setup -import io -from setuptools import setup, find_packages import pelicun -def read(*filenames, **kwargs): - """ - Utility function to read multiple files into a string +def read(*filenames, **kwargs) -> None: # noqa: ANN002, ANN003 + """Read multiple files into a string. + + Returns + ------- + str: The contents of the files joined by the specified separator. """ encoding = kwargs.get('encoding', 'utf-8') sep = kwargs.get('sep', '\n') buf = [] for filename in filenames: - with io.open(filename, encoding=encoding) as f: + with Path(filename).open(encoding=encoding) as f: buf.append(f.read()) return sep.join(buf) long_description = read('README.md') +# TODO(JVM): update documentation requirements, remove those no longer +# used. + setup( name='pelicun', version=pelicun.__version__, @@ -46,31 +86,39 @@ def read(*filenames, **kwargs): 'scipy>=1.7.0, <2.0', 'pandas>=1.4.0, <3.0', 'colorama>=0.4.0, <0.5.0', - 'numexpr>=2.8, <3.0' + 'numexpr>=2.8, <3.0', + 'jsonschema>=4.22.0, <5.0', # 'tables>=3.7.0', ], extras_require={ 'development': [ + 'codespell', 'flake8', 'flake8-bugbear', 'flake8-rst', + 'flake8-rst', 'flake8-rst-docstrings', + 'glob2', + 'jsonpath2', + 'jupyter', + 'jupytext', + 'mypy', + 'nbsphinx', + 'numpydoc', + 'pandas-stubs', + 'pydocstyle', 'pylint', 'pylint-pytest', - 'pydocstyle', - 'mypy', - 'black', 'pytest', 'pytest-cov', - 'glob2', - 'jupyter', - 'jupytext', + 'pytest-xdist', + 'rendre>0.0.14', + 'ruff==0.7.0', 'sphinx', 'sphinx-autoapi', - 'nbsphinx', - 'flake8-rst', - 'flake8-rst-docstrings', - 'pandas-stubs', + 'sphinx-rtd-theme', + 'sphinx_design', + 'sphinxcontrib-bibtex', 'types-colorama', ], },