diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml new file mode 100644 index 0000000..9012605 --- /dev/null +++ b/.github/workflows/release-please.yml @@ -0,0 +1,154 @@ +on: [push, pull_request] +name: release-please +jobs: + linting: + name: Linting + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install black ruff + - name: Autoformat with black + run: | + black . + - name: Lint with ruff + run: | + ruff check LoopDataConverter --fix + - uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "style: style fixes by ruff and autoformatting by black" + + continuous-integration: + name: Continuous integration ${{ matrix.os }} python ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ${{ fromJSON(vars.BUILD_OS)}} + python-version: ["3.9", "3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 + + with: + python-version: ${{ matrix.python-version }} + - name: Installing dependencies + shell: bash -l {0} + run: | + conda install -c conda-forge numpy scipy pytest dill beartype pandas geopandas shapely validators sqlite -y + - name: Building and install + shell: bash -l {0} + run: | + pip install . --user + - name: Testing + shell: bash -l {0} + run: | + pytest + + release-please: + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' + steps: + - uses: GoogleCloudPlatform/release-please-action@v4 + id: release + with: + release-type: python + package-name: LoopDataConverter + version-file: LoopDataConverter/version.py + outputs: + release_created: ${{ steps.release.outputs.release_created }} + # if a release is created then run the deploy scripts for github.io, conda, pypi and docker + + conda-deploy: + name: Building conda package for python ${{ matrix.os }}) + needs: "continuous-integration" + runs-on: ${{matrix.os}} + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + python-version: ["3.9", "3.10", "3.11", "3.12"] + steps: + - uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + python-version: ${{ matrix.python-version }} + + - uses: actions/checkout@v4 + - name: update submodules + # shell: bash -l {0} + run: | + git submodule update --init --recursive + - name: Conda build + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + shell: bash -l {0} + run: | + conda install -c conda-forge conda-build pytest numpy scipy pandas geopandas sqlite shapely validators anaconda-client -y + conda build -c anaconda -c conda-forge --output-folder conda conda + conda convert -p all conda/linux-64/*.tar.bz2 -f -o conda + conda install anaconda-client -y + - name: upload artifacts + uses: actions/upload-artifact@v4 + with: + name: conda-${{ matrix.os }}-${{ matrix.python-version }} + path: conda + + make_sdist: + needs: "continuous-integration" + name: Make SDist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build SDist + run: | + pip install build + python -m build + + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + upload_to_conda: + needs: ["release-please", "conda-deploy"] + runs-on: ${{matrix.os}} + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + python-version: ["3.9", "3.10", "3.11", "3.12"] + if: ${{ needs.release-please.outputs.release_created }} + steps: + - uses: actions/download-artifact@v4 + with: + name: conda-${{ matrix.os }}-${{ matrix.python-version }} + path: conda + - uses: conda-incubator/setup-miniconda@v3 + - name: upload all files to conda-forge + shell: bash -l {0} + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + run: | + conda install -c anaconda anaconda-client -y + anaconda upload --label main conda/*/*.tar.bz2 + + upload_to_pypi: + needs: ["release-please", "conda-deploy"] + runs-on: "ubuntu-latest" + + if: ${{ needs.release-please.outputs.release_created }} + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + skip_existing: true + verbose: true + user: ${{ secrets.PYPI_USERNAME }} + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.gitignore b/.gitignore index 4ac2212..4801bb1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ -LoopDataConverter.egg-info/dependency_links.txt -LoopDataConverter.egg-info/PKG-INFO +LoopDataConverter.egg-info/top_level.txt LoopDataConverter.egg-info/requires.txt LoopDataConverter.egg-info/SOURCES.txt -LoopDataConverter.egg-info/top_level.txt +LoopDataConverter.egg-info/PKG-INFO +LoopDataConverter.egg-info/dependency_links.txt diff --git a/LoopDataConverter/__init__.py b/LoopDataConverter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/LoopDataConverter/__pycache__/__init__.cpython-311.pyc b/LoopDataConverter/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..a34c479 Binary files /dev/null and b/LoopDataConverter/__pycache__/__init__.cpython-311.pyc differ diff --git a/LoopDataConverter/converters/__init__.py b/LoopDataConverter/converters/__init__.py new file mode 100644 index 0000000..8a97d5a --- /dev/null +++ b/LoopDataConverter/converters/__init__.py @@ -0,0 +1,14 @@ +from .base_converter import BaseConverter +from .ntgs_converter import NTGSConverter +from ..datatypes import SurveyName + +converter_map = { + SurveyName.NTGS: NTGSConverter, + SurveyName.GA: "", + SurveyName.GSQ: "", + SurveyName.GSWA: "", + SurveyName.GSSA: "", + SurveyName.GSV: "", + SurveyName.MRT: "", + SurveyName.GSNSW: "", +} diff --git a/LoopDataConverter/converters/base_converter.py b/LoopDataConverter/converters/base_converter.py new file mode 100644 index 0000000..f6ea5d0 --- /dev/null +++ b/LoopDataConverter/converters/base_converter.py @@ -0,0 +1,26 @@ +from abc import ABC, abstractmethod + + +class BaseConverter(ABC): + + def __init__(self): + self._type_label = "BaseTypeConverter" + + def type(self): + return self._type_label + + @abstractmethod + def convert_fold_map(self): + pass + + @abstractmethod + def convert_fault_map(self): + pass + + @abstractmethod + def convert_structure_map(self): + pass + + @abstractmethod + def convert(self): + pass diff --git a/LoopDataConverter/converters/conversion_manager.py b/LoopDataConverter/converters/conversion_manager.py new file mode 100644 index 0000000..1e6193b --- /dev/null +++ b/LoopDataConverter/converters/conversion_manager.py @@ -0,0 +1,44 @@ +from .ntgs_converter import NTGSConverter +from ..datatypes import SurveyName +from ..file_readers import LoopGisReader +from ..input import InputData + + +class LoopConverter: + """ + LoopConverter class use the LoopGisReader to look up the correct file + reader for the input file type and then converting the data to + Map2Loop or LoopStrucural formats using the adequate converter + """ + + def __init__(self, survey_name: SurveyName, data: InputData, layer: str = None): + self._fileData = data + self._layer = layer + self._survey_name = survey_name + self._converters = { + SurveyName.NTGS: NTGSConverter, + SurveyName.GA: "", + SurveyName.GSQ: "", + SurveyName.GSWA: "", + SurveyName.GSSA: "", + SurveyName.GSV: "", + SurveyName.GSNSW: "", + SurveyName.MRT: "", + } + + def read_file(self): + """ + read the file using the correct file reader + """ + file_reader = LoopGisReader(self._fileData) + file_reader.read(self._fileData, self._layer) + return file_reader.data + + def get_converter(self): + return self._converters[self._survey_name] + + def convert(self): + data = self.read_file() + converter = self.get_converter() + converter(data) + self.data = converter._data diff --git a/LoopDataConverter/converters/ntgs_converter.py b/LoopDataConverter/converters/ntgs_converter.py new file mode 100644 index 0000000..49ca4e5 --- /dev/null +++ b/LoopDataConverter/converters/ntgs_converter.py @@ -0,0 +1,68 @@ +# internal imports +from .base_converter import BaseConverter +from ..utils import ( + convert_dipdir_terms, + convert_dip_terms, + convert_tightness_terms, + convert_displacement_terms, +) + +# external imports +import pandas +import numpy + + +class NTGSConverter(BaseConverter): + # TODO: modify class to take fold, fault, and structure layers as arguments + def __init__(self, data: pandas.DataFrame): + self.raw_data = data.copy() + self._type_label = "NTGSConverter" + self._data = None + + def type(self): + return self._type_label + + def convert_fold_map(self): + # convert dip direction terms to degrees + self.raw_data["AxialPlaneDipDir"] = self.raw_data["AxialPlaneDipDir"].apply( + lambda x: convert_dipdir_terms(x) + ) + # convert dip terms to degrees + self.raw_data["AxialPlaneDip"] = self.raw_data["AxialPlaneDip"].apply( + lambda x: convert_dip_terms(x, type="fold") + ) + # convert tightness terms to degrees + self.raw_data["InterlimbAngle"] = self.raw_data["InterlimbAngle"].apply( + lambda x: convert_tightness_terms(x) + ) + + def convert_fault_map(self): + # convert dip direction terms to degrees + self.raw_data["DipDirection"] = self.raw_data["DipDirection"].apply( + lambda x: convert_dipdir_terms(x) + ) + # convert dip terms to degrees + self.raw_data["Dip"] = self.raw_data["Dip"].apply( + lambda x: convert_dip_terms(x, type="fault") + ) + self.raw_data["Displacement"] = self.raw_data["Displacement"].apply( + lambda x: convert_displacement_terms(x) + ) + + def convert_structure_map(self): + # discard any rows that has a dip value of -99 and does not have any esimated dip value + condition = (self.raw_data["Dip"] != -99) & (self.raw_data["DipEstimate"] != numpy.nan) + self.raw_data = self.raw_data[condition] + # convert dip estimate to float (average of the range) + condition = self.raw_data["Dip"] == -99 + self.raw_data.loc[condition, "DipEstimate"] = self.raw_data.loc[ + condition, "DipEstimate" + ].apply(lambda x: sum(map(float, x.split("-"))) / 2) + self.raw_data[condition, "Dip"] = self.raw_data[condition, "DipEstimate"] + + def convert(self): + self.convert_fold_map() + self.convert_fault_map() + self.convert_structure_map() + + self._data = self.raw_data.copy() diff --git a/LoopDataConverter/datatypes/__init__.py b/LoopDataConverter/datatypes/__init__.py new file mode 100644 index 0000000..5971574 --- /dev/null +++ b/LoopDataConverter/datatypes/__init__.py @@ -0,0 +1 @@ +from .enums import Datatype, SurveyName, Filetype diff --git a/LoopDataConverter/datatypes/__pycache__/__init__.cpython-311.pyc b/LoopDataConverter/datatypes/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..c3bb098 Binary files /dev/null and b/LoopDataConverter/datatypes/__pycache__/__init__.cpython-311.pyc differ diff --git a/LoopDataConverter/datatypes/__pycache__/enums.cpython-311.pyc b/LoopDataConverter/datatypes/__pycache__/enums.cpython-311.pyc new file mode 100644 index 0000000..66a7215 Binary files /dev/null and b/LoopDataConverter/datatypes/__pycache__/enums.cpython-311.pyc differ diff --git a/LoopDataConverter/datatypes/enums.py b/LoopDataConverter/datatypes/enums.py new file mode 100644 index 0000000..eafc6ab --- /dev/null +++ b/LoopDataConverter/datatypes/enums.py @@ -0,0 +1,31 @@ +from enum import IntEnum + + +class Datatype(IntEnum): + GEOLOGY = 0 + STRUCTURE = 1 + FAULT = 2 + FOLD = 3 + DTM = 4 + FAULT_ORIENTATION = 5 + + +class SurveyName(IntEnum): + + GA = 0 + NTGS = 1 + GSQ = 2 + GSWA = 3 + GSSA = 4 + GSV = 5 + MRT = 6 + GSNSW = 7 + + +class Filetype(IntEnum): + + CSV = 0 + GEOJSON = 1 + SHP = 2 + GPKG = 3 + ZIP = 4 diff --git a/LoopDataConverter/fields/__init__.py b/LoopDataConverter/fields/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/LoopDataConverter/fields/_ntgs/__init__.py b/LoopDataConverter/fields/_ntgs/__init__.py new file mode 100644 index 0000000..b209719 --- /dev/null +++ b/LoopDataConverter/fields/_ntgs/__init__.py @@ -0,0 +1 @@ +from ._ntgs_config import NtgsConfig diff --git a/LoopDataConverter/fields/_ntgs/_ntgs_config.py b/LoopDataConverter/fields/_ntgs/_ntgs_config.py new file mode 100644 index 0000000..e2a72a4 --- /dev/null +++ b/LoopDataConverter/fields/_ntgs/_ntgs_config.py @@ -0,0 +1,77 @@ +from ...datatypes.enums import Datatype + + +class NtgsConfig: + def __init__(self): + self.fold_config = ( + { + "structtype_column": "FoldEvent", + "fold_text": "FeatureCodeDesc", + "description_column": "Description", + "synform_text": "FoldType", + "foldname_column": "FoldName", + "objectid_column": "OBJECTID", + "tightness_column": "InterlimbAngle", + "axial_plane_dipdir_column": "AxialPlaneDipDir", + "axial_plane_dip_column": "AxialPlaneDip", + "interp_source_column": "InterpSource", + }, + ) + + self.fault_config = { + "structtype_column": "FaultType", + "fault_text": "'Normal', 'Reverse', 'Shear zone', 'Strike-slip', 'Thrust', 'Unknown'", + "dip_null_value": "-999", + "dipdir_flag": "num", + "dipdir_column": "DipDirection", + "dip_column": "Dip", + "orientation_type": "dip direction", + "dipestimate_column": "DipEstimate", + "dipestimate_text": "'NORTH_EAST','NORTH',,'NOT ACCESSED'", + "displacement_column": "Displacement", + "displacement_text": "'1m-100m', '100m-1km', '1km-5km', '>5km'", + "fault_length_column": "FaultLength", + "fault_length_text": "'Small (0-5km)', 'Medium (5-30km)', 'Large (30-100km)', 'Regional (>100km)', 'Unclassified'", + "name_column": "FaultName", + "objectid_column": "OBJECTID", + "interp_source_column": "InterpSource", + } + + self.geology_config = { + "unitname_column": "Formation", + "alt_unitname_column": "CODE", + "group_column": "GroupSuite", + "supergroup_column": "Supergroup", + "description_column": "LithDescription", + "minage_column": "AgeMin", + "maxage_column": "AgeMax", + "rocktype_column": "LithClass", + "alt_rocktype_column": "RockCategory", + "sill_text": "RockCategory", + "intrusive_text": "RockCategory", + "volcanic_text": "RockCategory", + "objectid_column": "OBJECTID", + "ignore_codes": ["cover"], + } + + self.structure_config = { + "orientation_type": "dip direction", + "dipdir_column": "DipDirection", + "dip_column": "Dip", + "description_column": "FeatureCodeDesc", + "bedding_text": "'Bedding', 'Cleavage', 'Faulting', 'Folding', 'Foliation', 'Geophysical', 'Igneous banding', 'Lineation'", + "overturned_column": "FeatureCodeDesc", + "overturned_text": "overturned", + "objectid_column": "ID", + "interp_source_column": "InterpSource", + } + + self.config_map = { + Datatype.GEOLOGY: self.geology_config, + Datatype.STRUCTURE: self.structure_config, + Datatype.FAULT: self.fault_config, + Datatype.FOLD: self.fold_config, + } + + def __getitem__(self, datatype): + return self.config_map[datatype] diff --git a/LoopDataConverter/file_readers/__init__.py b/LoopDataConverter/file_readers/__init__.py new file mode 100644 index 0000000..6bbc394 --- /dev/null +++ b/LoopDataConverter/file_readers/__init__.py @@ -0,0 +1 @@ +from ._file_readers import LoopGisReader, GeoDataFileReader, CSVFileReader diff --git a/LoopDataConverter/file_readers/__pycache__/__init__.cpython-311.pyc b/LoopDataConverter/file_readers/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..6b98aaf Binary files /dev/null and b/LoopDataConverter/file_readers/__pycache__/__init__.cpython-311.pyc differ diff --git a/LoopDataConverter/file_readers/__pycache__/_file_readers.cpython-311.pyc b/LoopDataConverter/file_readers/__pycache__/_file_readers.cpython-311.pyc new file mode 100644 index 0000000..f055b32 Binary files /dev/null and b/LoopDataConverter/file_readers/__pycache__/_file_readers.cpython-311.pyc differ diff --git a/LoopDataConverter/file_readers/__pycache__/file_readers.cpython-311.pyc b/LoopDataConverter/file_readers/__pycache__/file_readers.cpython-311.pyc new file mode 100644 index 0000000..c3673d3 Binary files /dev/null and b/LoopDataConverter/file_readers/__pycache__/file_readers.cpython-311.pyc differ diff --git a/LoopDataConverter/file_readers/_file_readers.py b/LoopDataConverter/file_readers/_file_readers.py new file mode 100644 index 0000000..c446c3d --- /dev/null +++ b/LoopDataConverter/file_readers/_file_readers.py @@ -0,0 +1,172 @@ +from abc import ABC, abstractmethod +from ..datatypes.enums import Datatype +import beartype +import pandas +import geopandas +import os +import validators + + +class BaseFileReader(ABC): + def __init__(self): + self.file_reader_label = "FileReaderBaseClass" + + def type(self): + return self.file_reader_label + + @beartype.beartype + @abstractmethod + def check_source_type(self, file_source: str): + assert validators.url(file_source) or os.path.isfile( + file_source + ), "Invalid file source, must be a valid URL or file path" + + @beartype.beartype + @abstractmethod + def get_file(self, file_source: str, layer: str = None): + pass + + @beartype.beartype + @abstractmethod + def save(self, file_path: str, file_extension: str = None): + pass + + @abstractmethod + def read(self): + pass + + +class CSVFileReader(BaseFileReader): + def __init__(self): + self.file_reader_label = "CSVFileReader" + self.file = None + self.data = None + + def type(self): + return self.file_reader_label + + @beartype.beartype + def check_source_type(self, file_source: str): + super().check_source_type(file_source) + + @beartype.beartype + def get_file(self, file_source: str, layer: str = None): + return pandas.read_csv(file_source) + + @beartype.beartype + def save(self, file_path: str): + self.data.to_csv(file_path) + + @beartype.beartype + def read(self, file_source: str): + self.check_source_type(file_source) + self.file = self.get_file(file_source) + self.data = pandas.DataFrame(self.file) + + +class GeoDataFileReader(BaseFileReader): + def __init__(self): + self.file_reader_label = "GeoDataFileReader" + self.file = None + self.data = None + + def type(self): + return self.file_reader_label + + @beartype.beartype + def check_source_type(self, file_source: str): + super().check_source_type(file_source) + + @beartype.beartype + def get_file(self, file_source: str, layer: str = None): + file_extension = os.path.splitext(file_source)[1] + + if file_extension in [".shp", ".geojson"]: + return geopandas.read_file(file_source) + + elif file_extension == ".gpkg": + assert layer is not None, "Layer name must be provided for GeoPackage files" + + return geopandas.read_file(file_source, layer=layer) + + else: + raise ValueError(f"Unsupported file format: {file_extension}") + + @beartype.beartype + def save(self, file_path: str, file_extension: str = None): + if file_extension == "geojson": + self.data.to_file(file_path, driver="GeoJSON") + + elif file_extension == "gpkg": + self.data.to_file(file_path, driver="GPKG") + + elif file_extension == "shp": + self.data.to_file(file_path) + + else: + raise ValueError(f"Unsupported file format: {file_extension}") + + @beartype.beartype + def read(self, file_source: str): + self.check_source_type(file_source) + self.file = self.get_file(file_source) + self.data = geopandas.GeoDataFrame(self.file) + + +class LoopGisReader: + def __init__(self, fileData, layer=None): + self._layer = layer + self._fileData = fileData + self._reader = [None] * len(Datatype) + self.file_reader_label = [None] * len(Datatype) + self._data = [None] * len(Datatype) + + def get_extension(self, file_source): + return os.path.splitext(file_source)[1] + + def assign_reader(self, file_source): + file_extension = self.get_extension(file_source) + + if file_extension == ".csv": + return CSVFileReader() + + elif file_extension in [".shp", ".geojson", ".gpkg"]: + return GeoDataFileReader() + + else: + raise ValueError(f"Unsupported file format: {file_extension}") + + def read(self, datatype: Datatype): + self._reader.read(self._fileData[Datatype.GEOLOGY], self._layer) + + return self._reader.data + + def __call__(self): + """ + Read all files in the input data + """ + + if self._fileData[Datatype.GEOLOGY] is not None: + self._reader[Datatype.GEOLOGY] = self.assign_reader(self._fileData[Datatype.GEOLOGY]) + self.file_reader_label[Datatype.GEOLOGY] = self._reader[Datatype.GEOLOGY].type() + self._data[Datatype.GEOLOGY] = self.read(Datatype.GEOLOGY) + + if self._fileData[Datatype.STRUCTURE] is not None: + self._reader[Datatype.STRUCTURE] = self.assign_reader( + self._fileData[Datatype.STRUCTURE] + ) + self.file_reader_label[Datatype.STRUCTURE] = self._reader[Datatype.STRUCTURE].type() + self._data[Datatype.STRUCTURE] = self.read(Datatype.STRUCTURE) + + if self._fileData[Datatype.FAULT] is not None: + self._reader[Datatype.FAULT] = self.assign_reader(self._fileData[Datatype.FAULT]) + self.file_reader_label[Datatype.FAULT] = self._reader[Datatype.FAULT].type() + self._data[Datatype.FAULT] = self.read(Datatype.FAULT) + + if self._fileData[Datatype.FOLD] is not None: + self._reader[Datatype.FOLD] = self.assign_reader(self._fileData[Datatype.FOLD]) + self.file_reader_label[Datatype.FOLD] = self._reader[Datatype.FOLD].type() + self._data[Datatype.FOLD] = self.read(Datatype.FOLD) + + def save(self, file_path, file_extension=None): + self._reader.save(file_path, file_extension) diff --git a/LoopDataConverter/input/__init__.py b/LoopDataConverter/input/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/LoopDataConverter/input/__pycache__/__init__.cpython-311.pyc b/LoopDataConverter/input/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..a2ac4c6 Binary files /dev/null and b/LoopDataConverter/input/__pycache__/__init__.cpython-311.pyc differ diff --git a/LoopDataConverter/input/__pycache__/input_data.cpython-311.pyc b/LoopDataConverter/input/__pycache__/input_data.cpython-311.pyc new file mode 100644 index 0000000..c8853bd Binary files /dev/null and b/LoopDataConverter/input/__pycache__/input_data.cpython-311.pyc differ diff --git a/LoopDataConverter/input/input_data.py b/LoopDataConverter/input/input_data.py new file mode 100644 index 0000000..214d3bb --- /dev/null +++ b/LoopDataConverter/input/input_data.py @@ -0,0 +1,39 @@ +from dataclasses import dataclass +from ..datatypes import Datatype + + +@dataclass +class InputData: + """Class to store input data for the loop data converter + + Attributes: + geology: Datatype.GEOLOGY = None + structure: Datatype.STRUCTURE = None + fault: Datatype.FAULT = None + fold: Datatype.FOLD = None + + """ + + geology: Datatype.GEOLOGY = None + structure: Datatype.STRUCTURE = None + fault: Datatype.FAULT = None + fold: Datatype.FOLD = None + + def __getitem__(self, datatype: Datatype): + """Method to get the the file directory of a datatype + + Parameters: + datatype (Datatype): The datatype to get the file directory of + + Returns: + The file directory of the datatype + """ + + return self.__dict__[datatype] + + +@dataclass +class OutputData(InputData): + + def __getitem__(self, datatype: Datatype): + return super().__getitem__(datatype) diff --git a/LoopDataConverter/utils/__init__.py b/LoopDataConverter/utils/__init__.py new file mode 100644 index 0000000..90e18d2 --- /dev/null +++ b/LoopDataConverter/utils/__init__.py @@ -0,0 +1,6 @@ +from .conversion import ( + convert_dip_terms, + convert_dipdir_cardinals, + convert_tightness_terms, + convert_displacement_terms, +) diff --git a/LoopDataConverter/utils/__pycache__/__init__.cpython-311.pyc b/LoopDataConverter/utils/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..9d6d65c Binary files /dev/null and b/LoopDataConverter/utils/__pycache__/__init__.cpython-311.pyc differ diff --git a/LoopDataConverter/utils/__pycache__/conversion.cpython-311.pyc b/LoopDataConverter/utils/__pycache__/conversion.cpython-311.pyc new file mode 100644 index 0000000..5457f40 Binary files /dev/null and b/LoopDataConverter/utils/__pycache__/conversion.cpython-311.pyc differ diff --git a/LoopDataConverter/utils/conversion.py b/LoopDataConverter/utils/conversion.py new file mode 100644 index 0000000..21a70c6 --- /dev/null +++ b/LoopDataConverter/utils/conversion.py @@ -0,0 +1,127 @@ +import numpy +import beartype + + +@beartype.beartype +def convert_dipdir_cardinals(cardinal: str): + """ + Convert cardinal directions to degrees. + + Parameters: + cardinal (str): The cardinal direction to convert. + + return (float): The cardinal direction in degrees. + """ + if cardinal == "N": + return 0.0 + elif cardinal == "NNE": + return 22.5 + elif cardinal == "NE": + return 45.0 + elif cardinal == "ENE": + return 67.5 + elif cardinal == "E": + return 90.0 + elif cardinal == "ESE": + return 112.5 + elif cardinal == "SE": + return 135.0 + elif cardinal == "SSE": + return 157.5 + elif cardinal == "S": + return 180.0 + elif cardinal == "SSW": + return 202.5 + elif cardinal == "SW": + return 225.0 + elif cardinal == "WSW": + return 247.5 + elif cardinal == "W": + return 270.0 + elif cardinal == "WNW": + return 292.5 + elif cardinal == "NW": + return 315.0 + elif cardinal == "NNW": + return 337.5 + else: + return numpy.nan + + +def convert_dip_terms(dip_term: str, type: str): + """ + Convert dip terms to degrees. + + Parameters: + dip_term (str): The dip term to convert. + + return (float): The dip term in degrees. + """ + if type == "fault": + if dip_term == "Vertical": + return 90.0 + elif dip_term == "Horizontal": + return 0.0 + elif dip_term == "Moderate": + return 45.0 + elif dip_term == "Steep": + return 75.0 + else: + return numpy.nan + + elif type == "fold": + if dip_term == "Upright": + return 90.0 + elif dip_term == "Recumbent": + return 0.0 + elif dip_term == "Inclined": + return 45.0 + elif dip_term == "Reclined": + return 75.0 + else: + return numpy.nan + + +def convert_tightness_terms(tightness_term: str): + """ + Convert tightness terms to degrees. + + Parameters: + tightness_term (str): The tightness term to convert. + + return (float): The tightness term in degrees, + which is the average of the interlimb angle range. + """ + if tightness_term == "gentle": + return 150.0 + elif tightness_term == "open": + return 95.0 + elif tightness_term == "close": + return 50.0 + elif tightness_term == "tight": + return 15.0 + elif tightness_term == "isoclinal": + return 0.0 + else: + return numpy.nan + + +def convert_displacement_terms(displacement_term: str): + """ + Convert displacement terms to meters. + + Parameters: + displacement_term (str): The displacement term to convert. + + return (float): The displacement term in meters. + """ + if displacement_term == "1m-100m": + return 50.5 + elif displacement_term == "100m-1km": + return 550.0 + elif displacement_term == "1km-5km": + return 3000.0 + elif displacement_term == ">5km": + return 5000.0 + else: + return numpy.nan diff --git a/LoopDataConverter/version.py b/LoopDataConverter/version.py new file mode 100644 index 0000000..65c45bf --- /dev/null +++ b/LoopDataConverter/version.py @@ -0,0 +1 @@ +__version__ = "0.1.0.beta.0" diff --git a/__pycache__/__init__.cpython-311.pyc b/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..2f73419 Binary files /dev/null and b/__pycache__/__init__.cpython-311.pyc differ diff --git a/conda/conda_build_config.yaml b/conda/conda_build_config.yaml new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/conda/conda_build_config.yaml @@ -0,0 +1 @@ + diff --git a/conda/meta.yaml b/conda/meta.yaml new file mode 100644 index 0000000..bef4916 --- /dev/null +++ b/conda/meta.yaml @@ -0,0 +1,50 @@ +{% set name = "LoopDataConverter" %} + +package: + name: "{{ name|lower }}" + version: "{{ environ.get('GIT_DESCRIBE_TAG', '') }}" + +source: + git_url: https://github.com/Loop3D/LoopDataConverter + +build: + number: 0 + script: "{{ PYTHON }} -m pip install ." + +requirements: + build: + - {{ compiler('cxx') }} + - numpy + - pandas + - geopandas + - shapely + - dill + - validators + - beartype + - sqlite + host: + - pip + - python + - setuptools + - numpy + run: + - numpy + - python>=3.9 +test: + import: + - numpy + - pandas + + +about: + home: "https://github.com/Loop3D/LoopDataConverter" + license: MIT + license_family: MIT + license_file: + summary: "Automatic conversion library of geological data (GIS) into Map2Loop and LoopStructural" + doc_url: "https://loop3d.github.io/LoopDataConverter/" + dev_url: + +extra: + recipe-maintainers: + - rabii-chaarani \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..159ea94 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,78 @@ +[tool.black] +line-length = 100 +skip-string-normalization = true +target-version = ['py39'] +exclude = '\.eggs|\.git|\.mypy_cache|\.tox|\.venv|_build|buck-out|build|dist|node_modules' +skip-magic-trailing-comma = true + +# Add an option to remove the trailing comma of separated lists +# but it doesn't exist because python devs don't like beautiful code. +# black specifically adds trailing commas due to git diff lengths +# which is an insufficient reason to destroy beautiful code making +# it look like an unfinished thought or an incomplete +[tool.blackdoc] +# From https://numpydoc.readthedocs.io/en/latest/format.html +# Extended discussion: https://github.com/pyvista/pyvista/pull/4129 +# The length of docstring lines should be kept to 75 characters to facilitate +# reading the docstrings in text terminals. +line-length = 75 + +[tool.build_sphinx] +source-dir = 'doc' +build-dir = './doc/build' +all_files = 1 + +[tool.upload_sphinx] +upload-dir = 'doc/build/html' + +[tool.pydocstyle] +match = '(?!coverage).*.py' +convention = "numpy" +add-ignore = ["D404"] + +[tool.codespell] +skip = '*.pyc,*.txt,*.gif,*.png,*.jpg,*.ply,*.vtk,*.vti,*.vtu,*.js,*.html,*.doctree,*.ttf,*.woff,*.woff2,*.eot,*.mp4,*.inv,*.pickle,*.ipynb,flycheck*,./.git/*,./.hypothesis/*,*.yml,doc/_build/*,./doc/images/*,./dist/*,*~,.hypothesis*,./doc/examples/*,*.mypy_cache/*,*cover,./tests/tinypages/_build/*,*/_autosummary/*' +quiet-level = 3 + + +[tool.ruff] +exclude = ['.git', 'pycache__', 'build', 'dist', 'doc/examples', 'doc/_build'] +line-length = 100 +indent-width = 4 +target-version = 'py39' + +[tool.ruff.lint] +external = ["E131", "D102", "D105"] +ignore = [ + # whitespace before ':' + "E203", + # line break before binary operator + # "W503", + # line length too long + "E501", + # do not assign a lambda expression, use a def + "E731", + # too many leading '#' for block comment + "E266", + # ambiguous variable name + "E741", + # module level import not at top of file + "E402", + # Quotes (temporary) + "Q0", + # bare excepts (temporary) + # "B001", "E722", + "E722", + # we already check black + # "BLK100", + # 'from module import *' used; unable to detect undefined names + "F403", +] +fixable = ["ALL"] +unfixable = [] +extend-select = ["B007", "B010", "C4", "F", "NPY", "PGH004", "RSE", "RUF100"] + +[tool.ruff.lint.flake8-comprehensions] +allow-dict-calls-with-keyword-arguments = true +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F401"] diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..38b3ef2 --- /dev/null +++ b/setup.py @@ -0,0 +1,45 @@ +import codecs +from setuptools import setup, find_packages +import os + +package_root = os.path.abspath(os.path.dirname(__file__)) + +version = {} +with open(os.path.join(package_root, "LoopDataConverter/version.py")) as fp: + exec(fp.read(), version) +version = version["__version__"] + +setup( + name="LoopDataConverter", + install_requires=["dill", "beartype", "numpy", "pandas", "geopandas", "shapely", "validators"], + python_requires=">=3.9", + description="Loop GIS data conversion library for LoopStructural and Map2Loop", + long_description=codecs.open("README.md", "r", "utf-8").read(), + author="Rabii Chaarani", + author_email="rabii.chaarani@monash.edu", + license=("MIT"), + url="https://github.com/Loop3D/LoopDataConverter", + classifiers=[ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Other Audience", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "License :: Free for non-commercial use", + "Natural Language :: English", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Topic :: Multimedia :: Graphics :: 3D Modeling", + "Topic :: Scientific/Engineering", + ], + version=version, + packages=find_packages(), + keywords=[ + "earth sciences", + "geology", + "3-D modelling", + "structural geology", + "GIS", + "geological mapping", + ], +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__pycache__/__init__.cpython-311.pyc b/tests/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..fa72753 Binary files /dev/null and b/tests/__pycache__/__init__.cpython-311.pyc differ diff --git a/tests/converters/__init__.py b/tests/converters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/file_readers/__init__.py b/tests/file_readers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/file_readers/__pycache__/__init__.cpython-311.pyc b/tests/file_readers/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..ce99d72 Binary files /dev/null and b/tests/file_readers/__pycache__/__init__.cpython-311.pyc differ diff --git a/tests/file_readers/__pycache__/test_file_readers.cpython-311-pytest-7.4.0.pyc b/tests/file_readers/__pycache__/test_file_readers.cpython-311-pytest-7.4.0.pyc new file mode 100644 index 0000000..1a5a4a7 Binary files /dev/null and b/tests/file_readers/__pycache__/test_file_readers.cpython-311-pytest-7.4.0.pyc differ diff --git a/tests/file_readers/test_file_readers.py b/tests/file_readers/test_file_readers.py new file mode 100644 index 0000000..989391d --- /dev/null +++ b/tests/file_readers/test_file_readers.py @@ -0,0 +1,173 @@ +import pytest +import geopandas +import pandas +import shapely +from LoopDataConverter.file_readers import GeoDataFileReader +import os +import tempfile + + +# build structures file +structures = [ + { + 'x': 2775.287768202244933, + 'y': 4330.15, + 'strike2': 45.00, + 'dip_2': 45.70, + 'id': 147.00, + 'sf': 's0', + }, + { + 'x': 3529.794754080061011, + 'y': 3091.192011237949828, + 'strike2': 288.50, + 'dip_2': 41.70, + 'id': 204.00, + 'sf': 's0', + }, + { + 'x': 7928.315269200518742, + 'y': 7234.561058065713951, + 'strike2': 48.80, + 'dip_2': 41.10, + 'id': 229.00, + 'sf': 's0', + }, + { + 'x': 8003.966104268994968, + 'y': 7421.634268009857806, + 'strike2': 48.80, + 'dip_2': 41.10, + 'id': 235.00, + 'sf': 's0', + }, + { + 'x': 6881.165236574942355, + 'y': 1213.128646564158771, + 'strike2': 299.10, + 'dip_2': 44.70, + 'id': 252.00, + 'sf': 's0', + }, + { + 'x': 3674.015651128655009, + 'y': 5266.677487068354822, + 'strike2': 41.20, + 'dip_2': 40.10, + 'id': 347.00, + 'sf': 's0', + }, + { + 'x': 3970.895076049027921, + 'y': 2944.223069901633608, + 'strike2': 273.00, + 'dip_2': 46.00, + 'id': 408.00, + 'sf': 's0', + }, +] + +for row in structures: + row['geometry'] = shapely.Point(row['x'], row['y']) + del row['x'], row['y'] + +shp_structures = geopandas.GeoDataFrame(structures, crs='epsg:7854') +# csv_structures = pandas.DataFrame(structures) +f_path = tempfile.mkdtemp() +shp_structures.to_file(os.path.join(f_path, "structures.shp")) +shp_structures.to_file(os.path.join(f_path, "structures.geojson"), driver="GeoJSON") + + +# csv_structures.to_csv(os.path.join(f_path, "structures.csv")) +# Fixtures for sample file sources +@pytest.fixture +def shp_file_source(): + # Assuming a sample .shp file exists for testing + return os.path.join(f_path, "structures.shp") + + +@pytest.fixture +def geojson_file_source(): + # Assuming a sample .geojson file exists for testing + return os.path.join(f_path, "structures.geojson") + + +# @pytest.fixture +# def gpkg_file_source(): +# # Assuming a sample .gpkg file exists for testing, with a layer name +# return "sample_data/sample.gpkg", "layer1" + + +@pytest.fixture +def invalid_file_source(): + return "sample_data/invalid.txt" + + +# Test Initialization +def test_initialization(): + reader = GeoDataFileReader() + assert reader.file_reader_label == "GeoDataFileReader" + + +# Test Check Source Type +def test_check_source_type_valid(shp_file_source, geojson_file_source): + reader = GeoDataFileReader() + # Assuming these calls do not raise an exception + reader.check_source_type(shp_file_source) + reader.check_source_type(geojson_file_source) + # reader.check_source_type(gpkg_file_source[0]) + + +def test_check_source_type_invalid(invalid_file_source): + reader = GeoDataFileReader() + with pytest.raises(AssertionError): + reader.check_source_type(invalid_file_source) + + +# Test Get File +@pytest.mark.parametrize("file_source", ["shp_file_source", "geojson_file_source"]) +def test_get_file(file_source, request): + reader = GeoDataFileReader() + file_source = request.getfixturevalue(file_source) + if isinstance(file_source, tuple): + file, layer = file_source + df = reader.get_file(file, layer) + else: + df = reader.get_file(file_source) + assert isinstance(df, geopandas.GeoDataFrame) + + +def test_get_file_unsupported(invalid_file_source): + reader = GeoDataFileReader() + with pytest.raises(ValueError): + reader.get_file(invalid_file_source) + + +# Test Read Method +def test_read_method_shp(shp_file_source): + reader = GeoDataFileReader() + reader.read(shp_file_source) + assert isinstance(reader.data, geopandas.GeoDataFrame) + + +def test_read_method_geojson(geojson_file_source): + reader = GeoDataFileReader() + reader.read(geojson_file_source) + assert isinstance(reader.data, geopandas.GeoDataFrame) + + +# Test Save Method +def test_save_method(geojson_file_source): + reader = GeoDataFileReader() + reader.read(geojson_file_source) + with tempfile.TemporaryDirectory() as tmpdirname: + save_path = os.path.join(tmpdirname, "output.geojson") + reader.save(save_path, "geojson") + assert os.path.exists(save_path) + + +def test_save_method_unsupported(geojson_file_source): + reader = GeoDataFileReader() + reader.read(geojson_file_source) + with pytest.raises(ValueError): + reader.save("output.unsupported") diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/utils/__pycache__/__init__.cpython-311.pyc b/tests/utils/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..0811caa Binary files /dev/null and b/tests/utils/__pycache__/__init__.cpython-311.pyc differ diff --git a/tests/utils/__pycache__/test_conversion.cpython-311-pytest-7.4.0.pyc b/tests/utils/__pycache__/test_conversion.cpython-311-pytest-7.4.0.pyc new file mode 100644 index 0000000..ab08b6d Binary files /dev/null and b/tests/utils/__pycache__/test_conversion.cpython-311-pytest-7.4.0.pyc differ diff --git a/tests/utils/test_conversion.py b/tests/utils/test_conversion.py new file mode 100644 index 0000000..0edc789 --- /dev/null +++ b/tests/utils/test_conversion.py @@ -0,0 +1,52 @@ +import pytest +from LoopDataConverter.utils import conversion + + +@pytest.mark.parametrize( + "cardinal,expected", + [ + ("N", 0.0), + ("NE", 45.0), + ("E", 90.0), + ("SE", 135.0), + ("S", 180.0), + ("SW", 225.0), + ("W", 270.0), + ("NW", 315.0), + ], +) +def test_convert_dipdir_cardinals(cardinal, expected): + assert conversion.convert_dipdir_cardinals(cardinal) == expected + + +@pytest.mark.parametrize( + "dip_term,type,expected", + [ + ("Vertical", "fault", 90.0), + ("Horizontal", "fault", 0.0), + ("Moderate", "fault", 45.0), + ("Steep", "fault", 75.0), + ("Upright", "fold", 90.0), + ("Recumbent", "fold", 0.0), + ("Inclined", "fold", 45.0), + ("Reclined", "fold", 75.0), + ], +) +def test_convert_dip_terms(dip_term, type, expected): + assert conversion.convert_dip_terms(dip_term, type) == expected + + +@pytest.mark.parametrize( + "tightness_term,expected", + [("gentle", 150.0), ("open", 95.0), ("close", 50.0), ("tight", 15.0), ("isoclinal", 0.0)], +) +def test_convert_tightness_terms(tightness_term, expected): + assert conversion.convert_tightness_terms(tightness_term) == expected + + +@pytest.mark.parametrize( + "displacement_term,expected", + [("1m-100m", 50.5), ("100m-1km", 550.0), ("1km-5km", 3000.0), (">5km", 5000.0)], +) +def test_convert_displacement_terms(displacement_term, expected): + assert conversion.convert_displacement_terms(displacement_term) == expected