fix: update to allow pandas 3.X versions with python 3.10+

.github/workflows/CDA-testing.yml

@@ -80,6 +80,13 @@ jobs:
           POETRY_INSTALLER_ONLY_BINARY: ':all:'
         run: poetry install --no-root
 
+      - name: Pin pandas 3.x for Python 3.11+
+        if: matrix.python-version != '3.9'
+        run: |
+          poetry env use "${{ matrix.python-version }}"
+          . "$(poetry env info --path)/bin/activate"
+          python -m pip install --upgrade --only-binary=:all: "pandas==3.0.6"
+
       # Run pytest and generate coverage report data.
       - name: Run Tests
         run: poetry run pytest tests/cda/ --doctest-modules --cov --cov-report=xml:out/coverage.xml

.github/workflows/testing.yml

@@ -19,7 +19,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.9', '3.10', '3.x']
+        python-version: ['3.9', '3.11', '3.x']
 
     steps:
       - uses: actions/checkout@v7
@@ -47,6 +47,13 @@ jobs:
           POETRY_INSTALLER_ONLY_BINARY: ':all:'
         run: poetry install
 
+      - name: Pin pandas 3.x for Python 3.11+
+        if: matrix.python-version != '3.9'
+        run: |
+          poetry env use "${{ matrix.python-version }}"
+          . "$(poetry env info --path)/bin/activate"
+          python -m pip install --upgrade --only-binary=:all: "pandas==3.0.6"
+
       # Run pytest and generate coverage report data.
       - name: Run Tests
         run: poetry run pytest tests/mock/ --doctest-modules --cov --cov-report=xml:out/coverage.xml

.gitignore

@@ -10,6 +10,7 @@ scripts
 .pytest_cache/
 
 .venv/
+.venv-pandas3/
 
 # Jupyter Notebook
 .ipynb_checkpoints

README.md

@@ -14,6 +14,17 @@ Python 3.9+
 pip install cwms-python
 ```
 
+For users on Python 3.11+ who want the pandas 3.x stack, install into a fresh virtual environment and upgrade pandas explicitly:
+
+```sh
+python3.11 -m venv .venv
+. .venv/bin/activate
+python -m pip install --upgrade pip
+pip install cwms-python "pandas>=3,<4"
+```
+
+This keeps the default package install stable while allowing a modern Python environment to use pandas 3.x.
+
 Then import the package:
 
 ```python

cwms/cwms_types.py

@@ -77,6 +77,7 @@ def timeseries_type(orig_json: JSON, value_json: JSON) -> DataFrame:
 
             if "date-time" in df.columns:
                 df["date-time"] = to_datetime(df["date-time"], unit="ms", utc=True)
+                df["date-time"] = df["date-time"].dt.as_unit("ms")
             return df
 
         def reorder_measurement_cols(df: DataFrame) -> DataFrame:

cwms/timeseries/timeseries.py

@@ -553,10 +553,10 @@ def timeseries_df_to_json(
             "value is a required column when posting data when posting as a dataframe"
         )
 
-    # make sure that dataTime column is in iso8601 formate.
-    df["date-time"] = pd.to_datetime(df["date-time"], utc=True).apply(
-        pd.Timestamp.isoformat
-    )
+    # make sure that date-time column is in ISO8601 format and keep CWMS
+    # millisecond precision to match the API and pandas 2/3 behavior.
+    df["date-time"] = pd.to_datetime(df["date-time"], utc=True).dt.as_unit("ms")
+    df["date-time"] = df["date-time"].apply(pd.Timestamp.isoformat)
     df = df.reindex(columns=["date-time", "value", "quality-code"])
 
     # Replace NaN/NA/NaT in value column with None so they serialize as JSON
@@ -664,27 +664,43 @@ def store_ts_ids(
     ts_data_all = data.copy()
     if "version_date" not in ts_data_all.columns:
         ts_data_all = ts_data_all.assign(version_date=pd.to_datetime(pd.Series([])))
+
+    def version_key(value: Any) -> str:
+        return "NaT" if pd.isna(value) else str(value)
+
+    def get_ts_group(
+        ts_id: str, version_date: str
+    ) -> Tuple[Optional[datetime], pd.DataFrame]:
+        if version_date == "NaT":
+            return (
+                None,
+                ts_data_all[
+                    (ts_data_all["ts_id"] == ts_id) & ts_data_all["version_date"].isna()
+                ],
+            )
+
+        version_date_dt = pd.to_datetime(version_date)
+        return (
+            version_date_dt,
+            ts_data_all[
+                (ts_data_all["ts_id"] == ts_id)
+                & (ts_data_all["version_date"] == version_date_dt)
+            ],
+        )
+
     unique_tsids = (
-        ts_data_all["ts_id"].astype(str) + ":" + ts_data_all["version_date"].astype(str)
+        ts_data_all["ts_id"].astype(str)
+        + ":"
+        + ts_data_all["version_date"].map(version_key)
     ).unique()
 
     errors: list[tuple[str, Exception]] = []
     with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
         futures = {}
         for unique_tsid in unique_tsids:
             ts_id, version_date = unique_tsid.split(":", 1)
-            if version_date != "NaT":
-                version_date_dt = pd.to_datetime(version_date)
-                ts_data = ts_data_all[
-                    (ts_data_all["ts_id"] == ts_id)
-                    & (ts_data_all["version_date"] == version_date_dt)
-                ]
-            else:
-                version_date_dt = None
-                ts_data = ts_data_all[
-                    (ts_data_all["ts_id"] == ts_id) & ts_data_all["version_date"].isna()
-                ]
-            if not data.empty:
+            version_date_dt, ts_data = get_ts_group(ts_id, version_date)
+            if not ts_data.empty:
                 future = executor.submit(
                     store_ts_ids, ts_data, ts_id, office_id, version_date_dt
                 )

cwms/timeseries/timeseries_group.py

@@ -138,20 +138,23 @@ def timeseries_group_df_to_json(
     if "attribute" not in df.columns:
         df["attribute"] = 0
 
-    # Replace NaN with None for optional columns
+    def normalize_optional_value(value: Any) -> Any:
+        return None if pd.isna(value) else value
+
+    # Replace NaN/NaT/NA with None for optional columns
     for column in optional_columns:
         if column in df.columns:
-            df[column] = df[column].where(pd.notnull(df[column]), None)
+            df[column] = df[column].map(normalize_optional_value)
 
     # Build the list of time-series entries
     assigned_time_series = df.apply(
         lambda entry: {
             "office-id": entry["office-id"],
             "timeseries-id": entry["timeseries-id"],
-            "alias-id": entry["alias-id"],
+            "alias-id": normalize_optional_value(entry["alias-id"]),
             "attribute": entry["attribute"],
             **(
-                {"ts-code": entry["ts-code"]}
+                {"ts-code": normalize_optional_value(entry["ts-code"])}
                 if "ts-code" in entry and pd.notna(entry["ts-code"])
                 else {}
             ),