pyarrow 25.0.1: PyArrow StringScalar equality with str literals returns False and null scalars evaluate truthy, while to_pandas degrades text columns to object dtype unless pandas metadata or ArrowDtype is present and distinct string types reject concatenation

{"case":{"caseId":"case:sha256:6104003c84bc2c222c1a66b9a570c32795088299eb0a7c35799d128725116dab","constraints":{"runtime":"python"},"contract":["assert isinstance(arr[0], pa.StringScalar) and (arr[0] == 'hello') is False","assert bool(pa.scalar('', type=pa.string())) is True and bool(pa.scalar(None, type=pa.string())) is True","assert arr.to_numpy() raises ArrowInvalid and arr.to_numpy(zero_copy_only=False).dtype == object","assert pa.Table.from_pydict({'text': ['a']}).to_pandas()['text'].dtype == object","assert table.replace_schema_metadata(None).to_pandas()['text'].dtype == object and table.to_pandas(types_mapper=pd.ArrowDtype)['text'].dtype.kind == 'U'","assert pa.concat_arrays([arr_utf8, arr_large]) raises ArrowInvalid until explicitly cast"],"goal":"PyArrow StringScalar equality with str literals returns False and null scalars evaluate truthy, while to_pandas degrades text columns to object dtype unless pandas metadata or ArrowDtype is present and distinct string types reject concatenation","kind":"HOW","packages":["pkg:pypi/pyarrow@25.0.1","pkg:pypi/pandas@2.3.3"],"schemaVersion":1,"symbols":["pyarrow.array","pyarrow.scalar","pyarrow.StringScalar","pyarrow.string","pyarrow.large_string","pyarrow.string_view","pyarrow.concat_arrays","pyarrow.Table.from_pydict","pyarrow.Table.from_pandas","pyarrow.Table.to_pandas","pyarrow.Array.to_numpy","pyarrow.Array.to_pylist"]},"contractCommand":["python","test/contract.py"],"environment":{"arch":"x64","ecosystem":"pypi","executionContext":"python","language":"python","os":"linux","packageManager":"pip","runtime":"python","schemaVersion":1},"license":"MIT-0","packages":["pkg:pypi/pyarrow@25.0.1","pkg:pypi/pandas@2.3.3"],"schemaVersion":1,"symbols":["pyarrow.array","pyarrow.scalar","pyarrow.StringScalar","pyarrow.string","pyarrow.large_string","pyarrow.string_view","pyarrow.concat_arrays","pyarrow.Table.from_pydict","pyarrow.Table.from_pandas","pyarrow.Table.to_pandas","pyarrow.Array.to_numpy","pyarrow.Array.to_pylist"],"verifierAdapter":"python@1"}

numpy==2.2.6
pandas==2.3.3
pyarrow==25.0.1
python-dateutil==2.9.0.post0
pytz==2026.3.post1
six==1.17.0
tzdata==2026.3

numpy==2.2.6
pandas==2.3.3
pyarrow==25.0.1
python-dateutil==2.9.0.post0
pytz==2026.3.post1
six==1.17.0
tzdata==2026.3

import numpy as np
import pandas as pd
import pyarrow as pa
import pyarrow.compute as pc


def test_string_scalar_comparison_and_types():
    arr = pa.array(["hello", "world"])
    s0 = arr[0]

    # Arrow indexing yields StringScalar, NOT a Python str
    assert isinstance(s0, pa.StringScalar)
    assert not isinstance(s0, str)

    # Comparing StringScalar directly to a Python str literal returns False
    assert (s0 == "hello") is False
    assert (s0 != "hello") is True

    # True equality requires .as_py() conversion or wrapping in pa.scalar()
    assert s0.as_py() == "hello"
    assert s0 == pa.scalar("hello")

    # Calling len() on a StringScalar raises TypeError
    try:
        len(s0)
        assert False, "Expected TypeError when calling len() on StringScalar"
    except TypeError:
        pass


def test_scalar_truthiness_and_nulls():
    s_empty = pa.scalar("", type=pa.string())
    s_null = pa.scalar(None, type=pa.string())

    # StringScalar instances are truthy in boolean context even when empty or null
    assert bool(s_empty) is True
    assert bool(s_null) is True

    # A null StringScalar is an object instance, so identity check against None fails
    assert (s_null is None) is False
    assert s_null.is_valid is False
    assert s_null.as_py() is None


def test_numpy_and_pylist_export():
    arr = pa.array(["apple", "banana", None])

    # Zero-copy view to NumPy is rejected because Arrow string buffers cannot map directly
    try:
        arr.to_numpy()
        assert False, "Expected ArrowInvalid when zero_copy_only=True"
    except pa.ArrowInvalid:
        pass

    # Copying to NumPy produces object dtype with Python str/None, not numpy unicode dtype
    np_arr = arr.to_numpy(zero_copy_only=False)
    assert np_arr.dtype == object
    assert np_arr.dtype != np.dtype("U")
    assert isinstance(np_arr[0], str)
    assert np_arr[2] is None

    # to_pylist produces native Python str and None
    py_list = arr.to_pylist()
    assert py_list == ["apple", "banana", None]
    assert isinstance(py_list[0], str)
    assert py_list[0] == "apple"
    assert py_list[2] is None


def test_pandas_roundtrip_and_metadata():
    # Pure Arrow Table without pandas metadata degrades text columns to object dtype
    t_pure = pa.Table.from_pydict({"text": ["alpha", "beta"]})
    df_pure = t_pure.to_pandas()
    assert df_pure["text"].dtype == object
    assert df_pure["text"].dtype != "string"
    assert not isinstance(df_pure["text"].dtype, pd.StringDtype)

    # DataFrame with string dtype retains string[python] only via embedded pandas metadata
    df_orig = pd.DataFrame({"s": pd.Series(["x", "y"], dtype="string")})
    t_from_df = pa.Table.from_pandas(df_orig)
    assert b"pandas" in (t_from_df.schema.metadata or {})
    df_restored = t_from_df.to_pandas()
    assert isinstance(df_restored["s"].dtype, pd.StringDtype)
    assert df_restored["s"].dtype.storage == "python"

    # Stripping metadata degrades the column back to object dtype
    t_no_meta = t_from_df.replace_schema_metadata(None)
    df_degraded = t_no_meta.to_pandas()
    assert df_degraded["s"].dtype == object
    assert not isinstance(df_degraded["s"].dtype, pd.StringDtype)

    # Explicit types_mapper is required to produce ArrowDtype string columns
    df_arrow = t_no_meta.to_pandas(types_mapper=pd.ArrowDtype)
    assert isinstance(df_arrow["s"].dtype, pd.ArrowDtype)
    assert df_arrow["s"].dtype.kind == "U"


def test_string_type_incompatibility():
    arr_utf8 = pa.array(["a", "b"], type=pa.string())
    arr_large = pa.array(["c", "d"], type=pa.large_string())
    arr_view = pa.array(["e", "f"], type=pa.string_view())

    # String type variants are not equal
    assert arr_utf8.type == pa.string()
    assert arr_large.type == pa.large_string()
    assert arr_view.type == pa.string_view()
    assert arr_utf8.type != arr_large.type
    assert arr_utf8.type != arr_view.type

    # Direct concatenation across distinct string types fails
    try:
        pa.concat_arrays([arr_utf8, arr_large])
        assert False, "Expected ArrowInvalid on concat of string and large_string"
    except pa.ArrowInvalid as exc:
        assert "identically typed" in str(exc)

    try:
        pa.concat_arrays([arr_utf8, arr_view])
        assert False, "Expected ArrowInvalid on concat of string and string_view"
    except pa.ArrowInvalid as exc:
        assert "identically typed" in str(exc)

    # Explicit cast allows concatenating different string representations
    concatenated = pa.concat_arrays([arr_utf8, arr_large.cast(pa.string())])
    assert len(concatenated) == 4
    assert concatenated.type == pa.string()
    assert concatenated.to_pylist() == ["a", "b", "c", "d"]


if __name__ == "__main__":
    test_string_scalar_comparison_and_types()
    test_scalar_truthiness_and_nulls()
    test_numpy_and_pylist_export()
    test_pandas_roundtrip_and_metadata()
    test_string_type_incompatibility()
    print("Contract tests passed successfully.")