Fix pandas 3.1 deprecations, require pandas>=2.3

petab/_utils.py

@@ -2,10 +2,20 @@
 
 from pathlib import Path
 
+import pandas as pd
 from pydantic import AnyUrl, TypeAdapter
 
 PathOrUrlAdapter = TypeAdapter(AnyUrl | Path)
 
+#: Extra keyword arguments for :func:`pandas.read_csv` to ensure lossless
+#: float parsing. ``float_precision`` is deprecated in pandas>=3.1,
+#: where round-trip parsing is the default.
+READ_CSV_FLOAT_KWARGS = (
+    {"float_precision": "round_trip"}
+    if tuple(map(int, pd.__version__.split(".")[:2])) < (3, 1)
+    else {}
+)
+
 
 def _generate_path(
     file_path: str | Path | AnyUrl,

petab/v1/conditions.py

@@ -6,6 +6,7 @@
 import numpy as np
 import pandas as pd
 
+from .._utils import READ_CSV_FLOAT_KWARGS
 from . import core, lint
 from .C import *
 
@@ -32,21 +33,20 @@ def get_condition_df(
 
     if isinstance(condition_file, str | Path):
         condition_file = pd.read_csv(
-            condition_file, sep="\t", float_precision="round_trip"
+            condition_file, sep="\t", **READ_CSV_FLOAT_KWARGS
         )
 
     lint.assert_no_leading_trailing_whitespace(
         condition_file.columns.values, "condition"
     )
 
     if not isinstance(condition_file.index, pd.RangeIndex):
-        condition_file.reset_index(
+        condition_file = condition_file.reset_index(
             drop=condition_file.index.name != CONDITION_ID,
-            inplace=True,
         )
 
     try:
-        condition_file.set_index([CONDITION_ID], inplace=True)
+        condition_file = condition_file.set_index([CONDITION_ID])
     except KeyError:
         raise KeyError(
             f"Condition table missing mandatory field {CONDITION_ID}."
@@ -90,7 +90,7 @@ def create_condition_df(
             raise ValueError("Invalid parameter ID: " + p)
         df[p] = np.nan
 
-    df.set_index(CONDITION_ID, inplace=True)
+    df = df.set_index(CONDITION_ID)
 
     return df
 

petab/v1/core.py

@@ -14,6 +14,7 @@
 import pandas as pd
 from pandas.api.types import is_string_dtype
 
+from .._utils import READ_CSV_FLOAT_KWARGS
 from . import yaml
 from .C import *
 
@@ -52,7 +53,7 @@ def get_simulation_df(simulation_file: str | Path) -> pd.DataFrame:
         Simulation DataFrame
     """
     return pd.read_csv(
-        simulation_file, sep="\t", index_col=None, float_precision="round_trip"
+        simulation_file, sep="\t", index_col=None, **READ_CSV_FLOAT_KWARGS
     )
 
 
@@ -94,7 +95,7 @@ def get_visualization_df(
             sep="\t",
             index_col=None,
             converters=types,
-            float_precision="round_trip",
+            **READ_CSV_FLOAT_KWARGS,
         )
     except pd.errors.EmptyDataError:
         warn(

petab/v1/mapping.py

@@ -5,6 +5,7 @@
 
 import pandas as pd
 
+from .._utils import READ_CSV_FLOAT_KWARGS
 from . import lint
 from .C import *
 from .models import Model
@@ -34,13 +35,12 @@ def get_mapping_df(
 
     if isinstance(mapping_file, str | Path):
         mapping_file = pd.read_csv(
-            mapping_file, sep="\t", float_precision="round_trip"
+            mapping_file, sep="\t", **READ_CSV_FLOAT_KWARGS
         )
 
     if not isinstance(mapping_file.index, pd.RangeIndex):
-        mapping_file.reset_index(
+        mapping_file = mapping_file.reset_index(
             drop=mapping_file.index.name != PETAB_ENTITY_ID,
-            inplace=True,
         )
 
     for col in MAPPING_DF_REQUIRED_COLS:
@@ -51,7 +51,7 @@ def get_mapping_df(
             mapping_file.reset_index()[col].values, col
         )
 
-    mapping_file.set_index([PETAB_ENTITY_ID], inplace=True)
+    mapping_file = mapping_file.set_index([PETAB_ENTITY_ID])
 
     return mapping_file
 

petab/v1/measurements.py

@@ -8,6 +8,7 @@
 import numpy as np
 import pandas as pd
 
+from .._utils import READ_CSV_FLOAT_KWARGS
 from . import core, lint, observables
 from .C import *
 
@@ -42,7 +43,7 @@ def get_measurement_df(
 
     if isinstance(measurement_file, str | Path):
         measurement_file = pd.read_csv(
-            measurement_file, sep="\t", float_precision="round_trip"
+            measurement_file, sep="\t", **READ_CSV_FLOAT_KWARGS
         )
 
     lint.assert_no_leading_trailing_whitespace(
@@ -94,7 +95,8 @@ def get_simulation_conditions(measurement_df: pd.DataFrame) -> pd.DataFrame:
     # We require NaN-containing rows, but they are ignored by `groupby`,
     # therefore replace them before
     simulation_conditions = (
-        measurement_df.fillna("")
+        measurement_df[grouping_cols]
+        .fillna("")
         .groupby(grouping_cols)
         .size()
         .reset_index()[grouping_cols]
@@ -129,7 +131,9 @@ def get_rows_for_condition(
     # check for equality in all grouping cols
     if PREEQUILIBRATION_CONDITION_ID in condition:
         row_filter = (
-            measurement_df[PREEQUILIBRATION_CONDITION_ID].fillna("")
+            measurement_df[PREEQUILIBRATION_CONDITION_ID]
+            .astype("object")
+            .fillna("")
             == condition[PREEQUILIBRATION_CONDITION_ID]
         ) & row_filter
     if SIMULATION_CONDITION_ID in condition:
@@ -251,7 +255,7 @@ def measurements_have_replicates(measurement_df: pd.DataFrame) -> bool:
         ],
     )
     return np.any(
-        measurement_df.fillna("").groupby(grouping_cols).size().values - 1
+        measurement_df.groupby(grouping_cols, dropna=False).size().values - 1
     )
 
 

petab/v1/observables.py

@@ -7,6 +7,7 @@
 
 import pandas as pd
 
+from .._utils import READ_CSV_FLOAT_KWARGS
 from . import core, lint
 from .C import *
 from .math import sympify_petab
@@ -39,21 +40,20 @@ def get_observable_df(
 
     if isinstance(observable_file, str | Path):
         observable_file = pd.read_csv(
-            observable_file, sep="\t", float_precision="round_trip"
+            observable_file, sep="\t", **READ_CSV_FLOAT_KWARGS
         )
 
     lint.assert_no_leading_trailing_whitespace(
         observable_file.columns.values, "observable"
     )
 
     if not isinstance(observable_file.index, pd.RangeIndex):
-        observable_file.reset_index(
+        observable_file = observable_file.reset_index(
             drop=observable_file.index.name != OBSERVABLE_ID,
-            inplace=True,
         )
 
     try:
-        observable_file.set_index([OBSERVABLE_ID], inplace=True)
+        observable_file = observable_file.set_index([OBSERVABLE_ID])
     except KeyError:
         raise KeyError(
             f"Observable table missing mandatory field {OBSERVABLE_ID}."

petab/v1/parameters.py

@@ -14,6 +14,7 @@
 import numpy as np
 import pandas as pd
 
+from .._utils import READ_CSV_FLOAT_KWARGS
 from . import conditions, core, lint, measurements, observables
 from .C import *
 from .models import Model
@@ -59,7 +60,7 @@ def get_parameter_df(
         parameter_df = parameter_file
     elif isinstance(parameter_file, str | Path):
         parameter_df = pd.read_csv(
-            parameter_file, sep="\t", float_precision="round_trip"
+            parameter_file, sep="\t", **READ_CSV_FLOAT_KWARGS
         )
     elif isinstance(parameter_file, Iterable):
         dfs = [get_parameter_df(x) for x in parameter_file if x]
@@ -78,13 +79,12 @@ def get_parameter_df(
     )
 
     if not isinstance(parameter_df.index, pd.RangeIndex):
-        parameter_df.reset_index(
+        parameter_df = parameter_df.reset_index(
             drop=parameter_file.index.name != PARAMETER_ID,
-            inplace=True,
         )
 
     try:
-        parameter_df.set_index([PARAMETER_ID], inplace=True)
+        parameter_df = parameter_df.set_index([PARAMETER_ID])
     except KeyError as e:
         raise KeyError(
             f"Parameter table missing mandatory field {PARAMETER_ID}."
@@ -240,7 +240,7 @@ def create_parameter_df(
             OBJECTIVE_PRIOR_PARAMETERS: "",
         }
     )
-    df.set_index([PARAMETER_ID], inplace=True)
+    df = df.set_index([PARAMETER_ID])
 
     # For model parameters, set nominal values as defined in the model
     for parameter_id in df.index:

petab/v1/simplify.py

@@ -23,8 +23,7 @@ def remove_nan_measurements(problem: Problem):
     """Drop any measurements that are NaN"""
     problem.measurement_df = problem.measurement_df[
         ~problem.measurement_df[MEASUREMENT].isna()
-    ]
-    problem.measurement_df.reset_index(inplace=True, drop=True)
+    ].reset_index(drop=True)
 
 
 def remove_unused_observables(problem: Problem):
@@ -103,8 +102,7 @@ def condition_parameters_to_parameter_table(problem: Problem):
         }
         for parameter_id, value in replacements.items()
     ]
-    rows = pd.DataFrame(rows)
-    rows.set_index(PARAMETER_ID, inplace=True)
+    rows = pd.DataFrame(rows).set_index(PARAMETER_ID)
 
     if problem.parameter_df is None:
         problem.parameter_df = rows

petab/v1/visualize/data_overview.py

@@ -83,7 +83,10 @@ def get_data_per_observable(measurement_df: pd.DataFrame) -> pd.DataFrame:
     )
 
     # Add row and column sums
-    data_per_observable.loc["SUM", :] = data_per_observable.sum(axis=0).values
+    sum_row = "SUM" if len(index) == 1 else ("SUM", "")
+    data_per_observable.loc[sum_row, :] = data_per_observable.sum(
+        axis=0
+    ).values
     data_per_observable["SUM"] = data_per_observable.sum(axis=1).values
 
     data_per_observable = data_per_observable.astype(int)

petab/v1/visualize/plotting.py

@@ -61,8 +61,7 @@ def __init__(
         conditions_: np.ndarray | pd.Series | None,
         data_to_plot: pd.DataFrame | None = None,
     ):
-        self.data_to_plot = data_to_plot
-        self.data_to_plot.sort_index(inplace=True)
+        self.data_to_plot = data_to_plot.sort_index()
 
         self.conditions = conditions_
         if self.conditions is not None:
@@ -76,7 +75,7 @@ def __init__(
         if isinstance(self.conditions, np.ndarray):
             self.conditions.sort()
         elif isinstance(self.conditions, pd.Series):
-            self.conditions.sort_index(inplace=True)
+            self.conditions = self.conditions.sort_index()
 
     def add_x_offset(self, offset) -> None:
         """

petab/v2/conditions.py

@@ -6,6 +6,7 @@
 
 import pandas as pd
 
+from .._utils import READ_CSV_FLOAT_KWARGS
 from ..v1.lint import assert_no_leading_trailing_whitespace
 
 __all__ = [
@@ -27,7 +28,7 @@ def get_condition_df(
 
     if isinstance(condition_file, str | Path):
         condition_file = pd.read_csv(
-            condition_file, sep="\t", float_precision="round_trip"
+            condition_file, sep="\t", **READ_CSV_FLOAT_KWARGS
         )
 
     assert_no_leading_trailing_whitespace(

petab/v2/experiments.py

@@ -4,6 +4,8 @@
 
 import pandas as pd
 
+from .._utils import READ_CSV_FLOAT_KWARGS
+
 __all__ = ["get_experiment_df", "write_experiment_df"]
 
 
@@ -22,7 +24,7 @@ def get_experiment_df(
 
     if isinstance(experiments_file, str | Path):
         experiments_file = pd.read_csv(
-            experiments_file, sep="\t", float_precision="round_trip"
+            experiments_file, sep="\t", **READ_CSV_FLOAT_KWARGS
         )
 
     return experiments_file

petab/v2/petab1to2.py

@@ -231,15 +231,16 @@ def create_experiment_id(sim_cond_id: str, preeq_cond_id: str) -> str:
         measurement_df = v1.get_measurement_df(get_src_path(measurement_file))
         # if there is already an experiment ID column, we rename it
         if v2.C.EXPERIMENT_ID in measurement_df.columns:
-            measurement_df.rename(
+            measurement_df = measurement_df.rename(
                 columns={v2.C.EXPERIMENT_ID: f"experiment_id_{uuid4()}"},
-                inplace=True,
             )
         # add pre-eq condition id if not present or convert to string
         #  for simplicity
         if v1.C.PREEQUILIBRATION_CONDITION_ID in measurement_df.columns:
-            measurement_df.fillna(
-                {v1.C.PREEQUILIBRATION_CONDITION_ID: ""}, inplace=True
+            measurement_df[v1.C.PREEQUILIBRATION_CONDITION_ID] = (
+                measurement_df[v1.C.PREEQUILIBRATION_CONDITION_ID]
+                .astype("object")
+                .fillna("")
             )
         else:
             measurement_df[v1.C.PREEQUILIBRATION_CONDITION_ID] = ""
@@ -362,7 +363,7 @@ def v1v2_condition_df(
     condition_df = condition_df.copy().reset_index()
     with suppress(KeyError):
         # conditionName was dropped in PEtab v2
-        condition_df.drop(columns=[v1.C.CONDITION_NAME], inplace=True)
+        condition_df = condition_df.drop(columns=[v1.C.CONDITION_NAME])
 
     condition_df = condition_df.melt(
         id_vars=[v1.C.CONDITION_ID],
@@ -458,7 +459,7 @@ def get_noise_formula(row):
                 stacklevel=4,
             )
 
-        df.drop(columns=[v1.C.OBSERVABLE_TRANSFORMATION], inplace=True)
+        df = df.drop(columns=[v1.C.OBSERVABLE_TRANSFORMATION])
 
     def extract_placeholders(row: pd.Series, type_: str) -> str:
         """Extract placeholders from observable formula."""
@@ -588,12 +589,11 @@ def update_prior(row) -> tuple[str, str | None]:
         )
 
     # rename objectivePrior* to prior*
-    df.rename(
+    df = df.rename(
         columns={
             v1.C.OBJECTIVE_PRIOR_TYPE: v2.C.PRIOR_DISTRIBUTION,
             v1.C.OBJECTIVE_PRIOR_PARAMETERS: v2.C.PRIOR_PARAMETERS,
         },
-        inplace=True,
         errors="ignore",
     )
     # some columns were dropped in PEtab v2
@@ -610,13 +610,12 @@ def update_prior(row) -> tuple[str, str | None]:
             "Parameter scales are not supported in PEtab v2.",
             stacklevel=9,
         )
-    df.drop(
+    df = df.drop(
         columns=[
             v1.C.INITIALIZATION_PRIOR_TYPE,
             v1.C.INITIALIZATION_PRIOR_PARAMETERS,
             v1.C.PARAMETER_SCALE,
         ],
-        inplace=True,
         errors="ignore",
     )