# geoML - machine learning models for geospatial data
# Copyright (C) 2020 Ítalo Gomes Gonçalves
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR a PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import os as _os
import glob as _glob
import pandas as _pd
import numpy as _np
import geoml.data as _data
import geoml.data.drillhole as _drillhole
[docs]
def walker():
"""
Walker lake dataset.
Returns
-------
walker_point : geoml.data.PointData
Dataset with 470 samples.
walker_grid : geoml.data.Grid2D
Full data.
"""
path = _os.path.dirname(__file__)
path_walker = _os.path.join(path, "sample_data/walker.dat")
path_walker_ex = _os.path.join(path, "sample_data/walker_ex.dat")
walker_sample = _pd.read_table(path_walker, na_values=-999) * 1.0
walker_ex = _pd.read_table(path_walker_ex, sep=",")
walker_point = _data.PointData(walker_sample, ["X", "Y"])
walker_point.add_continuous_variable("V", walker_sample["V"].values)
walker_point.add_continuous_variable("U", walker_sample["U"].values)
walker_grid = _data.Grid2D(start=[1, 1], n=[260, 300], step=[1, 1])
walker_grid.add_continuous_variable("V", walker_ex["V"].values)
walker_grid.add_continuous_variable("U", walker_ex["U"].values)
return walker_point, walker_grid
[docs]
def ararangua():
"""
Drillhole data from Araranguá town, Brazil.
Returns
-------
ara_dh : geoml.drillhole.DrillholeData
A dataset with 13 drillholes and one lithology table, named "lito".
"""
path = _os.path.dirname(__file__)
file = _os.path.join(path, "sample_data/Araranguá.xlsx")
ara_lito = _pd.read_excel(file, sheet_name="Lito")
ara_collar = _pd.read_excel(file, sheet_name="Collar")
# there is no survey: these holes are vertical, which is stated here
# rather than left to the default
ara_collar["Dip"] = 90.0
ara_collar["Azimuth"] = 0.0
ara_dh = _drillhole.DrillholeData(
ara_collar, hole="Hole ID", x="X", y="Y", z="Z", length="Length",
dip="Dip", azimuth="Azimuth")
ara_dh.add_intervals(
"lito", ara_lito, hole="Hole ID", fr="From", to="To",
categorical=["Lito", "Formation", "Layer"])
return ara_dh
[docs]
def macpass(path):
"""
Drillhole data from the Macmillan Pass (Macpass) project, Yukon, Canada.
This data is NOT distributed with geoML. It is published by Fireweed
Metals Corp. at https://fireweedmetals.com/macpass-project/ and is subject
to the terms at https://fireweedmetals.com/macpass-disclaimer/, which the
user accepts by downloading it. Among other things, those terms state that
users agree to use the dataset for their own purposes only, and that the
accuracy, completeness and reliability of the data are not guaranteed.
Download the four CSV files yourself and pass the directory holding them.
Note that this database records a downward hole with a negative dip, the
opposite of geoML's convention, which is why the object below is built
with `dip_positive_down=False`.
Parameters
----------
path : str
Directory with the MPA_Collar, MPA_Survey, MPA_Interp and
MPA_Samples_BD CSV files.
Returns
-------
macpass_dh : geoml.drillhole.DrillholeData
559 drillholes, with the interval tables "litho" (interpreted rock
code) and "assay" (Ag, Pb, Zn and bulk density).
"""
def find(pattern):
matches = _glob.glob(_os.path.join(path, pattern))
if len(matches) == 0:
raise FileNotFoundError(
f"no file matching {pattern} in {path}; download the data "
f"from https://fireweedmetals.com/macpass-project/")
return sorted(matches)[-1]
# the samples file carries a byte order mark
collar = _pd.read_csv(find("MPA_Collar_*.csv"))
survey = _pd.read_csv(find("MPA_Survey_*.csv"))
litho = _pd.read_csv(find("MPA_Interp_*.csv"))
assay = _pd.read_csv(find("MPA_Samples_BD_*.csv"), encoding="utf-8-sig")
macpass_dh = _drillhole.DrillholeData(
collar, survey, hole="HoleID", x="Easting", y="Northing",
z="Elevation", length="Length_m", dip="Dip", azimuth="Azimuth",
depth="Depth_m", dip_positive_down=False)
macpass_dh.add_intervals(
"litho", litho, hole="holeid", fr="from", to="to",
categorical="Code", ignore=["Code_Long", "Code_Description"])
macpass_dh.add_intervals(
"assay", assay, hole="HoleID", fr="From_m", to="To_m",
grades=["Ag_ppm", "Pb_pct", "Zn_pct"], density="BD_tonnes_m3",
flags=["Method", "LowRecovery_<=85pct"], ignore="Comment")
return macpass_dh
[docs]
def example_fold():
"""
Example directional data.
Returns
-------
point : geoml.data.PointData
Some coordinates with two rock labels.
dirs : geoml.data.DirectionalData
Structural measurements_a representing a fold.
"""
ex_point = _pd.DataFrame(
{"X": _np.array([
# rock a
25, 40, 60, 85, 89, 76, 66, 74, 64, 50, 50, 31, 21, 25, 16, 30,
# rock b
5, 10, 45, 50, 55, 75, 90, 91, 74, 64,
76, 66, 50, 50, 29, 19, 14, 25,
# boundary
15, 20, 30, 50, 65, 75, 90, 25, 50, 65, 75,
]),
"Y": _np.array([
# rock a
25, 60, 50, 15, 19, 11, 21, 49, 64, 51, 84, 64, 34, 11, 16, 45,
# rock b
50, 80, 10, 30, 10, 75, 90, 21, 9, 19,
51, 66, 49, 86, 66, 36, 14, 9,
# boundary
15, 35, 65, 85, 65, 50, 20, 10, 50, 20, 10,
]),
"rock": _np.concatenate([_np.repeat("a", 16),
_np.repeat("b", 18),
_np.repeat("boundary", 11)])})
ex_point["label_1"] = _np.concatenate(
[_np.repeat("a", 16),
_np.repeat("b", 18),
_np.repeat("a", 11)])
ex_point["label_2"] = _np.concatenate(
[_np.repeat("a", 16),
_np.repeat("b", 18),
_np.repeat("b", 11)])
ex_dir = _pd.DataFrame(
{"X": _np.array([40, 50, 70, 90, 30, 20, 20]),
"Y": _np.array([40, 85, 70, 30, 50, 60, 10]),
"strike": _np.array([30, 90, 145, 145, 30, 30, 30])})
ex_dir["azimuth"] = ex_dir["strike"] - 90
point = _data.PointData(ex_point, ["X", "Y"])
point.add_rock_type_variable("rock", labels=["a", "b"],
measurements_a=ex_point["label_1"].values,
measurements_b=ex_point["label_2"].values)
vals = _np.ones(ex_point.shape[0])
vals[ex_point["label_1"] == "b"] = -1
vals[ex_point["label_1"] != ex_point["label_2"]] = 0
point.add_continuous_variable("rock_num", vals)
dirs = _data.DirectionalData.from_azimuth(
ex_dir, ["X", "Y"], "strike"
)
normals = _data.DirectionalData.from_azimuth(
ex_dir, ["X", "Y"], "azimuth"
)
return point, dirs, normals
[docs]
def sunspot_number():
"""
Sunspot number data.
This data is downloaded from the Royal Observatory of Belgium
SILSO website (http://sidc.be/silso/home), and is distributed under
the CC BY-NC4.0 license (https://goo.gl/PXrLYd).
Returns
-------
out - dict
Dict containing the processed and original data.
"""
yearly_df = _pd.read_table(
"http://sidc.be/silso/INFO/snytotcsv.php",
sep=";", header=None)
yearly_df = yearly_df.set_axis(["year", "sn", "sn_std",
"n_obs", "definitive"],
axis="columns")
yearly = _data.PointData(yearly_df, "year")
yearly.add_continuous_variable("sn", yearly_df["sn"].values)
monthly_df = _pd.read_table(
"http://sidc.oma.be/silso/INFO/snmtotcsv.php",
sep=";", header=None)
monthly_df = monthly_df.set_axis(["year", "month", "year_frac", "sn", "sn_std",
"n_obs", "definitive"],
axis="columns")
monthly_df["idx"] = _np.arange(1, monthly_df.shape[0] + 1, dtype=float)
monthly = _data.PointData(monthly_df, "idx")
monthly.add_continuous_variable("sn", monthly_df["sn"].values)
daily_df = _pd.read_table("http://sidc.oma.be/silso/INFO/sndtotcsv.php",
sep=";", header=None)
daily_df = daily_df.set_axis(["year", "month", "day", "year_frac", "sn", "sn_std",
"n_obs", "definitive"],
axis="columns")
daily_df["idx"] = _np.arange(1, daily_df.shape[0] + 1, dtype=float)
daily = _data.PointData(daily_df, "idx")
daily.add_continuous_variable("sn", daily_df["sn"].values)
out = {"points": {"yearly": yearly,
"monthly": monthly,
"daily": daily},
"data_frames": {"yearly": yearly_df,
"monthly": monthly_df,
"daily": daily_df}}
return out
[docs]
def andrade():
"""
Structural measurements in Cerro do Andrade, Caçapava do Sul, Brazil.
Returns
-------
planes : geoml.data.DirectionalData
Directions parallel to the foliation planes.
normals : geoml.data.DirectionalData
Normals to the foliation planes.
"""
path = _os.path.dirname(__file__)
file = _os.path.join(path, "sample_data/andrade.txt")
raw_data = _pd.read_table(file, sep=",")
planes = _data.DirectionalData.from_planes(
raw_data, ["X", "Y", "Z"], "azimuth", "dip"
)
normals = _data.DirectionalData.from_normals(
raw_data, ["X", "Y", "Z"], "azimuth", "dip"
)
return planes, normals, raw_data
[docs]
def jura():
"""
Jura mountains dataset (Goovaerts, 1997).
Returns
-------
jura_train : geoml.data.PointData
Training data (2 categorical variables and 7 continuous).
jura_val : geoml.data.PointData
Validation data (2 categorical variables and 7 continuous).
"""
elements = ["Cd", "Co", "Cr", "Cu", "Ni", "Pb", "Zn"]
path = _os.path.dirname(__file__)
file_a = _os.path.join(path, "sample_data/jura_train.csv")
train_df = _pd.read_csv(file_a)
landuse_labels = _np.unique(train_df["Landuse"])
rock_labels = _np.unique(train_df["Rock"])
jura_train = _data.PointData(train_df, coordinates=["Xloc", "Yloc"])
jura_train.add_categorical_variable("Landuse", landuse_labels,
train_df["Landuse"])
jura_train.add_categorical_variable("Rock", rock_labels, train_df["Rock"])
# for el in elements:
# jura_train.add_continuous_variable(el, train_df[el])
jura_train.add_vector_variable('Elements', elements, train_df.loc[:, elements].values)
file_b = _os.path.join(path, "sample_data/jura_val.csv")
val_df = _pd.read_csv(file_b)
jura_val = _data.PointData(val_df, coordinates=["Xloc", "Yloc"])
jura_val.add_categorical_variable("Landuse", landuse_labels,
val_df["Landuse"])
jura_val.add_categorical_variable("Rock", rock_labels, val_df["Rock"])
# for el in elements:
# jura_val.add_continuous_variable(el, val_df[el])
jura_val.add_vector_variable('Elements', elements, val_df.loc[:, elements].values)
return jura_train, jura_val
[docs]
def arctic_lake():
"""
A compositional dataset.
Returns
-------
arctic_lake_data : geoml.data.PointData
References
----------
Pawlowsky-Glahn, V., Egozcue, J. J., & Tolosana-Delgado, R. (2015).
Modeling and Analysis of Compositional Data. John Wiley & Sons.
"""
path = _os.path.dirname(__file__)
file = _os.path.join(path, "sample_data/Arctic_lake.csv")
raw_data = _pd.read_csv(file)
arctic_lake_data = _data.PointData(raw_data, ["Depth (m)"])
arctic_lake_data.add_compositional_variable(
"comp", labels=['Sand', 'Silt', 'Clay'],
measurements=raw_data.values[:, :3] / 100)
return arctic_lake_data