ecto 3.14.1: Validate a plain map with Ecto.Changeset alone: an embedded_schema, no Repo, no database and no ecto_sql

{"buildCommand":["mix","deps.compile","--no-deps-check"],"case":{"caseId":"case:sha256:ea95d7c80806ff0a2d465ae6172e07487fb7995b62517a40e44e3e6188a06c72","constraints":{"runtime":"elixir"},"contract":["assert Ecto.Changeset validates with :ecto alone: ecto_sql is not installed and Ecto.Adapters.SQL will not load, while the embedded_schema reports no source table and no prefix where an otherwise identical schema/2 block reports its table name","assert embedded_schema's own default primary key is an autogenerated binary_id that nothing outside a Repo insert ever fills in, so it stays nil, which is why the schema turns it off","assert cast/4 keeps only the permitted fields and silently drops the rest, adding no error for a real-but-unpermitted field or for a key that is not a field at all, while changeset.params still holds the raw attempt and permitting that same field would have accepted it with no error either","assert permitting a field the schema does not have raises ArgumentError, so a typo in the permitted list fails loudly while an unexpected param never does","assert cast substitutes the field's default for an empty or whitespace-only string, so validate_required reports can't be blank, and that the substituted value is the declared default rather than nil, which silently resets a field that has one","assert a value that survives the emptiness check is stored untrimmed, so only the check trims","assert validate_required runs no blank check of its own and reuses changeset.empty_values, so casting with empty_values: [] lets an empty string pass validation while a validator that inspects the value still catches it","assert passing a function in :empty_values now raises ArgumentError and the two-arity :trim_values option is the replacement","assert validate_required decides presence with get_field, so a field carrying a non-nil schema default can never be reported missing, on a failing changeset or a passing one","assert a type failure from cast is reported as is invalid with validation: :cast, leaves the field out of changes, and suppresses the later validators for that field","assert apply_action/2 returns {:ok, struct} or {:error, changeset} with :action stamped onto the error changeset, apply_action!/2 raises Ecto.InvalidChangesetError, and apply_changes/1 returns the struct even when the changeset is invalid","assert validate_number stores a %{number} placeholder plus interpolation metadata while validate_format's message is already complete, so a raw error message is not displayable","assert traverse_errors interpolates and groups the repeated keys of changeset.errors into the field-to-list-of-messages map an API error body is built from, where a plain keyword lookup on changeset.errors returns only the most recent of a field's several errors","assert put_change writes to changes regardless of the permitted list and regardless of the validators that already ran, is a no-op when the value equals the one on the data, and force_change/3 is the override"],"goal":"Validate a plain map with Ecto.Changeset alone: an embedded_schema, no Repo, no database and no ecto_sql","kind":"HOW","packages":["pkg:hex/ecto@3.14.1"],"schemaVersion":1,"symbols":["Ecto.Schema.embedded_schema","Ecto.Changeset.cast","Ecto.Changeset.validate_required","Ecto.Changeset.validate_number","Ecto.Changeset.validate_format","Ecto.Changeset.validate_length","Ecto.Changeset.empty_values","Ecto.Changeset.field_missing?","Ecto.Changeset.apply_action","Ecto.Changeset.apply_changes","Ecto.Changeset.traverse_errors","Ecto.Changeset.put_change","Ecto.Changeset.force_change","Ecto.InvalidChangesetError"]},"contractCommand":["mix","test","--no-deps-check"],"environment":{"arch":"x64","ecosystem":"hex","executionContext":"elixir","language":"elixir","os":"linux","packageManager":"mix","runtime":"elixir","runtimeVersion":"1","schemaVersion":1},"license":"MIT-0","packages":["pkg:hex/ecto@3.14.1"],"schemaVersion":1,"symbols":["Ecto.Schema.embedded_schema","Ecto.Changeset.cast","Ecto.Changeset.validate_required","Ecto.Changeset.validate_number","Ecto.Changeset.validate_format","Ecto.Changeset.validate_length","Ecto.Changeset.empty_values","Ecto.Changeset.field_missing?","Ecto.Changeset.apply_action","Ecto.Changeset.apply_changes","Ecto.Changeset.traverse_errors","Ecto.Changeset.put_change","Ecto.Changeset.force_change","Ecto.InvalidChangesetError"],"verifierAdapter":"hex@1"}

defmodule CsxEcto.Signup do
  @moduledoc """
  Ecto.Changeset used as a standalone validator for a plain map: an
  embedded_schema, no Repo, no ecto_sql, no database of any kind.

  embedded_schema is the piece that makes this work. It defines the same
  struct and the same `__schema__/1` reflection a table-backed schema gets,
  minus the table, so every cast and validate function behaves identically
  while nothing ever opens a connection. The only Ecto function in the whole
  library that needs a Repo is the one that talks to a Repo.
  """

  use Ecto.Schema
  import Ecto.Changeset

  # embedded_schema defaults to @primary_key {:id, :binary_id, autogenerate: true}.
  # Autogeneration happens inside a Repo insert, so with no Repo that field
  # would be a permanent nil pretending to be an id. Turning it off keeps the
  # struct honest about what it is.
  @primary_key false
  embedded_schema do
    field :name, :string
    field :email, :string
    field :age, :integer
    field :accepted_terms, :boolean

    # :role is a real field that is deliberately missing from @permitted.
    # This is the security-relevant half of cast/4 and the reason the
    # permitted list exists at all: a request that sends "role" => "admin"
    # gets no error, no change, and no warning. See changeset/1.
    field :role, :string, default: "member"
  end

  @permitted [:name, :email, :age, :accepted_terms]

  @doc """
  Build a changeset from raw string-keyed params, the shape a JSON body
  arrives in.

  cast/4 does three things at once, and conflating them is where the
  surprises come from: it filters params down to the permitted list, it
  converts each remaining value to the field's declared type, and it replaces
  a value it considers empty with the field's declared default before storing
  anything. Only the second of those can produce an error. Filtering is
  silent, and emptying looks like the param was never sent. The replacement is
  the declared default and not nil, which only becomes visible once a field
  has one.
  """
  def changeset(params) when is_map(params) do
    %__MODULE__{}
    |> cast(params, @permitted)
    |> validate_required([:name, :email])
    |> validate_format(:email, ~r/^[^@\s]+@[^@\s]+$/)
    |> validate_number(:age, greater_than_or_equal_to: 18)
  end

  @doc """
  Render `changeset.errors` as the `%{field => [message]}` map an API
  response wants.

  Ecto stores an error as `{message, opts}` where the message still contains
  `%{placeholder}` markers and the opts carry the values, so that a
  translation layer can pick a different string per locale and per count.
  That means the raw message is not displayable: printing
  `elem(error, 0)` puts a literal "%{number}" in front of a user. Something
  has to do this substitution, and outside Phoenix nothing does it for you.
  """
  def error_map(%Ecto.Changeset{} = changeset) do
    Ecto.Changeset.traverse_errors(changeset, fn {message, opts} ->
      Regex.replace(~r"%{(\w+)}", message, fn _match, key ->
        opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string()
      end)
    end)
  end
end

defmodule CsxEcto.MixProject do
  use Mix.Project

  def project do
    [
      app: :csx_ecto,
      version: "0.1.0",
      elixir: "~> 1.18",
      deps: deps()
    ]
  end

  def application, do: [extra_applications: []]

  defp deps do
    # :ecto only. Ecto.Changeset, Ecto.Schema and embedded_schema all live in
    # this package; ecto_sql is the separate package that adds Ecto.Adapters.SQL
    # and the migration tooling, and nothing here needs it. The contract
    # asserts that ecto_sql really is absent, so "you can validate without a
    # database" is measured rather than claimed.
    #
    # ecto pulls in :decimal and :telemetry as required dependencies. :jason is
    # an optional dependency, needed only for Ecto's JSON-typed fields, so it
    # never lands in the lock here.
    [{:ecto, "3.14.1"}]
  end
end

%{
  "decimal": {:hex, :decimal, "3.1.1", "430d87b04011ce6cbd4fd205be758311a81f87d552d40904abd00f015935b1d0", [:mix], [], "hexpm", "c5f25f2ced74a0587d03e6023f595db8e924c9d3922c8c8ffd9edfc4498cf1f6"},
  "ecto": {:hex, :ecto, "3.14.1", "7b740d87bdf45996aa0c2c2e081640906f10caa7ce5ba328fd294c7d49d0cc6f", [:mix], [{:decimal, "~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "24b991956796700f467d0a3ef3d303138a3ef9ddddf8b98f43758ee067b20a30"},
  "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"},
}

defmodule CsxEcto.TableBacked do
  use Ecto.Schema

  # The contrast for the first test. schema/2 is the same block with a table
  # name in front of it, and that name is what __schema__(:source) reports.
  # Declaring one still needs no Repo — the Repo dependency starts at the
  # first call that reads or writes the table, not at the schema.
  @primary_key false
  schema "signups" do
    field :name, :string
  end
end

defmodule CsxEcto.DefaultPk do
  use Ecto.Schema

  # What Signup would get if it did not set @primary_key false.
  embedded_schema do
    field :name, :string
  end
end

defmodule CsxEcto.SignupTest do
  use ExUnit.Case

  import Ecto.Changeset

  alias CsxEcto.DefaultPk
  alias CsxEcto.Signup
  alias CsxEcto.TableBacked

  @good %{"name" => "Ada", "email" => "ada@example.com", "age" => "36"}

  test "validation runs with ecto alone, and the schema has no table behind it" do
    # ecto_sql is a different package. Everything this file exercises comes
    # from :ecto, so the adapter layer is simply not here — no Repo to define,
    # no adapter to configure, no connection pool to start.
    assert Application.spec(:ecto, :vsn) != nil
    assert Application.spec(:ecto_sql) == nil
    refute Code.ensure_loaded?(Ecto.Adapters.SQL)

    # embedded_schema is the marker for "struct, no table", and the reflection
    # says so: a schema/2 block answers this same question with its table name.
    assert Signup.__schema__(:source) == nil
    assert TableBacked.__schema__(:source) == "signups"
    assert Signup.__schema__(:prefix) == nil
    assert Signup.__schema__(:fields) == [:name, :email, :age, :accepted_terms, :role]
    assert Signup.__schema__(:type, :age) == :integer

    assert Signup.changeset(@good).valid?
  end

  test "embedded_schema still declares an autogenerated id, which is why Signup turns it off" do
    # The default primary key is a :binary_id marked for autogeneration, and
    # autogeneration happens inside Repo.insert. With no Repo nothing ever
    # fills it in, so the field is a permanent nil that looks like an id in
    # every inspect and every JSON body until someone tries to key on it.
    assert DefaultPk.__schema__(:primary_key) == [:id]
    assert DefaultPk.__schema__(:autogenerate_id) == {:id, :id, :binary_id}

    blank = %DefaultPk{}
    assert blank.id == nil
    assert apply_changes(cast(blank, %{"name" => "Ada"}, [:name])).id == nil

    assert Signup.__schema__(:primary_key) == []
  end

  test "cast keeps the permitted fields and silently drops everything else" do
    changeset =
      Signup.changeset(%{
        "name" => "Ada",
        "email" => "ada@example.com",
        "age" => "36",
        "accepted_terms" => "true",
        "role" => "admin",
        "nickname" => "not a field at all"
      })

    # Type conversion is the visible half: strings from JSON become the
    # declared types.
    assert changeset.changes == %{
             name: "Ada",
             email: "ada@example.com",
             age: 36,
             accepted_terms: true
           }

    # The invisible half. "role" is a real field, "nickname" is not a field at
    # all, and both are treated identically: dropped, with no error and no
    # trace in the result.
    assert changeset.errors == []
    refute Map.has_key?(changeset.changes, :role)
    assert apply_changes(changeset).role == "member"

    # The permitted list is the entire defence, and it is a whitelist for this
    # reason: add :role to it by accident and the same request escalates, with
    # valid? still true and errors still empty. Nothing reports that anything
    # happened either way, so a test that only checks valid? cannot tell these
    # two changesets apart.
    escalated = cast(%Signup{}, %{"role" => "admin"}, [:name, :role])
    assert escalated.changes == %{role: "admin"}
    assert escalated.errors == []
    assert escalated.valid?

    # The dropped values are not destroyed, only ignored. changeset.params
    # holds the raw map as handed in, so an audit log or a rate limiter can
    # still see that "role" was attempted.
    assert changeset.params["role"] == "admin"
    assert changeset.params["nickname"] == "not a field at all"
  end

  test "permitting a field that does not exist raises, unlike sending one" do
    # The asymmetry is worth internalising. A typo in your own permitted list
    # is a loud ArgumentError at cast time. A key the attacker adds to the
    # request is silence. Ecto validates the list you wrote, never the params
    # you received.
    assert_raise ArgumentError, fn ->
      cast(%Signup{}, %{}, [:nickname])
    end
  end

  test "cast empties an empty string to the field default, so required fails" do
    changeset = Signup.changeset(%{"name" => "", "email" => "ada@example.com"})

    refute changeset.valid?

    # The field is absent from changes rather than present-and-empty: cast
    # compared "" against its :empty_values list and substituted the field's
    # default, which for :name is nil and therefore equal to the data, so
    # nothing was recorded as a change at all. This is why `changes[:name]`
    # reads nil in the debugger and people conclude the param never arrived.
    refute Map.has_key?(changeset.changes, :name)
    assert changeset.params["name"] == ""
    assert {"can't be blank", [validation: :required]} = changeset.errors[:name]

    # "the default", not "nil", is the actual rule, and the two only coincide
    # for a field declared without one. Over existing data an empty string
    # nils out a field with no default and silently resets a field that has
    # one — sending "" for :role does not clear it, it demotes an admin to the
    # schema default.
    assert cast(%Signup{name: "Ada"}, %{"name" => ""}, [:name]).changes == %{name: nil}
    assert cast(%Signup{role: "admin"}, %{"role" => ""}, [:role]).changes == %{role: "member"}

    # The list cast compares against is literally [""], and the comparison is
    # made after trimming, so a whitespace-only string is emptied as well.
    # Only the emptiness test trims — a value that survives it is stored
    # exactly as it was sent.
    assert Ecto.Changeset.empty_values() == [""]
    assert Signup.changeset(%{"name" => "   ", "email" => "ada@example.com"}).errors[:name]
    assert cast(%Signup{}, %{"name" => "  Ada  "}, [:name]).changes == %{name: "  Ada  "}

    # Measured, and it refutes the obvious reading of the pipeline:
    # validate_required does not run a blank check of its own. It asks
    # whether the value is nil or a member of changeset.empty_values — the
    # very list cast recorded on the changeset — so switching cast's emptying
    # off disarms validate_required at the same time, and "" is accepted as a
    # present value. The two functions look independent and are not.
    kept = cast(%Signup{}, %{"name" => ""}, [:name], empty_values: [])
    assert kept.empty_values == []
    assert kept.changes == %{name: ""}
    assert validate_required(kept, [:name]).valid?
    assert validate_required(kept, [:name]).errors == []

    # What does catch it is a validator that looks at the value, because the
    # field is now present enough to reach one.
    assert validate_length(kept, :name, min: 2).errors[:name]

    # The trimming is a separate option from the emptiness list as of 3.14.
    # Older advice to hand :empty_values a predicate now raises rather than
    # being ignored, which is the good outcome; a two-arity :trim_values
    # function is the replacement, and disabling it keeps whitespace.
    assert_raise ArgumentError, fn ->
      cast(%Signup{}, %{"name" => "  "}, [:name], empty_values: [&(&1 == "")])
    end

    untrimmed = cast(%Signup{}, %{"name" => "   "}, [:name], trim_values: fn _type, value -> value end)
    assert untrimmed.changes == %{name: "   "}
  end

  test "validate_required reads through to the struct, so a default hides it" do
    changeset = Signup.changeset(%{"email" => "ada@example.com"})

    # Presence is decided by get_field, which falls back to changeset.data
    # when there is no change. :role carries a schema default of "member", so
    # it is never missing and validate_required(:role) cannot fail whatever
    # the params contain. Requiring a field with a non-nil default is dead
    # config.
    assert field_missing?(changeset, :name)
    refute field_missing?(changeset, :role)
    assert validate_required(changeset, [:role]).errors[:role] == nil

    # Nor is the error merely being masked by the one :name already produced.
    # On a changeset with nothing else wrong, requiring :role adds nothing:
    # the check cannot fail, whatever the params were.
    good = Signup.changeset(@good)
    assert validate_required(good, [:role]).valid?
    assert validate_required(good, [:role]).errors == []
  end

  test "a type failure is a cast error, not a validation error" do
    changeset = Signup.changeset(%{"name" => "Ada", "email" => "ada@example.com", "age" => "old"})

    refute changeset.valid?

    # cast reports its own failures with validation: :cast and the target
    # type, and the field is left out of changes. Every validator after it is
    # then skipped for that field, so validate_number never sees "old" and
    # never adds a second error. Errors from cast and errors from validators
    # look alike in the errors list and are produced by completely different
    # machinery.
    assert changeset.errors[:age] == {"is invalid", [type: :integer, validation: :cast]}
    refute Map.has_key?(changeset.changes, :age)
    assert length(changeset.errors) == 1
  end

  test "valid? is a field, apply_action is the gate, apply_changes is neither" do
    ok = Signup.changeset(@good)
    assert ok.valid?
    assert ok.action == nil

    # apply_action/2 is the no-Repo equivalent of Repo.insert: it returns the
    # ok/error tuple a `with` chain expects. The action atom it takes is only a
    # label — nothing is inserted anywhere — but it must be given.
    assert {:ok, %Signup{} = applied} = apply_action(ok, :insert)
    assert applied.name == "Ada"
    assert applied.age == 36
    assert applied.role == "member"

    bad = Signup.changeset(%{"email" => "nope"})
    refute bad.valid?
    assert {:error, errored} = apply_action(bad, :insert)

    # The error branch does not hand back the changeset you passed in. It
    # stamps :action onto it, which is the flag Phoenix uses to decide whether
    # to render errors, and which makes `assert {:error, ^bad}` fail on a
    # changeset that is otherwise identical.
    assert errored.action == :insert
    assert errored.changes == bad.changes
    refute errored == bad

    # apply_changes/1 is the trap next to it. It applies the changes to the
    # struct and returns it whether or not the changeset is valid, with no
    # tuple to pattern match and nothing to ignore. Reaching for it because
    # "I just want the struct" is how invalid data gets past validation
    # entirely.
    assert %Signup{email: "nope", name: nil} = apply_changes(bad)

    assert_raise Ecto.InvalidChangesetError, fn -> apply_action!(bad, :insert) end
  end

  test "errors carry interpolation metadata, not finished sentences" do
    changeset = Signup.changeset(%{"name" => "Ada", "email" => "not-an-email", "age" => "12"})

    refute changeset.valid?

    # validate_number stores the bound in the opts and leaves a %{number}
    # marker in the message. Rendering elem(error, 0) directly is the bug:
    # the user is shown a literal "%{number}".
    assert changeset.errors[:age] ==
             {"must be greater than or equal to %{number}",
              [validation: :number, kind: :greater_than_or_equal_to, number: 18]}

    # validate_format has nothing to interpolate, so its message happens to be
    # complete. That inconsistency is what makes the bug survive review: half
    # the messages look fine without a renderer.
    assert changeset.errors[:email] == {"has invalid format", [validation: :format]}
  end

  test "traverse_errors is what turns the changeset into an API body" do
    changeset = Signup.changeset(%{"email" => "not-an-email", "age" => "12"})

    # Keys are field atoms, values are always lists — one field can collect
    # several errors — and this is the shape to build a JSON error body from.
    assert Signup.error_map(changeset) == %{
             name: ["can't be blank"],
             email: ["has invalid format"],
             age: ["must be greater than or equal to 18"]
           }

    # The values are lists because a field really can collect several errors,
    # and changeset.errors stores them as repeated keys. The keyword-list
    # lookup everyone reaches for returns the most recently added one and
    # silently discards the rest, so `changeset.errors[:email]` here reports
    # the length problem and never mentions the format problem. Grouping is
    # the job traverse_errors is doing, and it is why walking changeset.errors
    # with a keyword lookup per field loses errors.
    both = validate_length(changeset, :email, max: 3)
    assert Keyword.get_values(both.errors, :email) |> length() == 2
    assert {"should be at most %{count} character(s)", _} = both.errors[:email]

    assert Signup.error_map(both)[:email] ==
             ["should be at most 3 character(s)", "has invalid format"]
    assert Signup.error_map(Signup.changeset(@good)) == %{}
  end

  test "put_change goes around the permitted list and around the validators" do
    escalated = Signup.changeset(@good) |> put_change(:role, "admin")

    # :role is not in @permitted and put_change does not care. It writes to
    # changes directly, so it is the one call that reintroduces the mass
    # assignment problem cast/4 exists to prevent — worth grepping for in
    # review whenever the value came from a request.
    assert escalated.changes.role == "admin"
    assert escalated.valid?
    assert apply_changes(escalated).role == "admin"

    # It skips validation as thoroughly as it skips filtering. Validators are
    # ordinary functions that ran earlier in the pipeline; a change added
    # after them is never inspected, and the changeset still reports valid?.
    late = Signup.changeset(@good) |> put_change(:age, 12)
    assert late.changes.age == 12
    assert late.valid?
    assert late.errors == []

    # Running the same validator again does catch it, which is the fix when a
    # put_change has to stay: the pipeline order is the semantics.
    assert validate_number(late, :age, greater_than_or_equal_to: 18).errors[:age]

    # And put_change is a no-op when the value already equals the one on the
    # struct, because a changeset records differences rather than assignments.
    # Forcing the write is what force_change/3 is for.
    unchanged = put_change(Signup.changeset(@good), :role, "member")
    refute Map.has_key?(unchanged.changes, :role)
    assert force_change(Signup.changeset(@good), :role, "member").changes.role == "member"
  end
end

ExUnit.start()