feat: Durable operation identity

CHANGELOG.md

@@ -1,5 +1,46 @@
 # Revision history for dataframe
 
+## Unreleased
+
+### Breaking changes
+* Operators are identified by tag, not name. `UnaryOp`/`BinaryOp` expose
+  `unaryId`/`binaryId :: OpRef …` (`Builtin` tag, `Named` text or
+  `Anonymous` label); `unaryName`/`binaryName` are now plain functions.
+  `MkUnaryOp`/`MkBinaryOp` take an `OpRef` instead of a name, and
+  `AggStrategy` constructors take an `OpRef AggId`.
+* `liftDecorated`/`lift2Decorated` build `Named` ops, which never use a
+  builtin kernel; library code uses the new `liftBuiltin`/`lift2Builtin`.
+* `CastWith`/`CastExprWith` take an `OpRef CastId` instead of a text tag.
+* `kExtremes` takes a `Bool` (largest/smallest) instead of a tag and a
+  comparator, so the two can no longer disagree.
+* Every `DataFrame.Functions` combinator that does not take a Haskell
+  function is a builtin; `lift`, `lift2`, `nullLift`, `nullLift2`,
+  `reduce`, `whenPresent`, `whenBothPresent` and `bind` stay anonymous.
+* Typed `not` prints as `~`, like the untyped `not`.
+* `DataFrame.Functions` holds only expression combinators: `sanitize`,
+  `isHaskellIdentifier`, `isReservedId` and `isVarId` move to
+  `DataFrame.TH.Identifier` (dataframe-th), and `optionalToDoubleVector` to
+  `DataFrame.Internal.Statistics`.
+
+### Additions
+* `DataFrame.Typed` gains `cast`, `castWithDefault`, `castEither`,
+  `unsafeCast` (schema-checked column), `eq`, `lt`, `gt`, `leq`, `geq`,
+  `and`, `or`, `countDistinct`, `topK`, `bottomK`, `isNull`, `isNotNull`,
+  `contains` and `length`, completing parity with `DataFrame.Functions`.
+
+### Fixes
+* A user function named like a builtin (e.g. `"add"`) no longer runs the
+  builtin's kernel instead of the user's function.
+* `eqExpr` no longer equates different `lift`/`lift2` functions, `fromMaybe`
+  with different defaults, or `(^)` with `(^^)`.
+* The predicate simplifier no longer merges conditions on different
+  user functions, and no longer treats `fromMaybe True` as false on nulls.
+* The grouped moments fast path no longer drops a user function inside `sum`.
+* `medianMaybe` prints as `medianMaybe`, not `meanMaybe`.
+* Parquet row-group pruning recognises `F.and`/`.&&` conjunctions.
+* The expression wire format decodes `logBase`, `stddev` and
+  `countDistinct`; `(^^)` encodes as `powFrac`.
+
 ## 3.7.0.0
 
 * `dataframe-viz`: column names are escaped in Vega-Lite field references

dataframe-core/dataframe-core.cabal

@@ -76,6 +76,7 @@ library
                         -- Expressions: syntax, operators, simplification.
                         DataFrame.Internal.Expression
                         DataFrame.Internal.Expression.Operators
+                        DataFrame.Internal.Expression.OpId
                         DataFrame.Internal.Expression.Reflect
                         DataFrame.Internal.Expression.Operators.Nullable
                         DataFrame.Internal.Expression.Simplify

dataframe-core/src-internal/DataFrame/Internal/Aggregation/Plan.hs

@@ -2,7 +2,7 @@
 {-# LANGUAGE ExplicitNamespaces #-}
 {-# LANGUAGE GADTs #-}
 {-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE TypeApplications #-}
 
@@ -28,10 +28,15 @@
     getColumn,
  )
 import DataFrame.Internal.Expression (
+    AggId (..),
     AggStrategy (..),
-    BinaryOp (binaryCommutative, binaryName),
+    BinOpId (..),
+    BinaryOp (binaryId),
     Expr (..),
+    OpRef (..),
     SomeExpr (..),
+    UnOpId (..),
+    UnaryOp (unaryId),
  )
 import Type.Reflection (Typeable, typeRep)
 
@@ -49,27 +54,25 @@
 -- | Plan a named output expression, or 'Nothing' if the shape is unsupported.
 planAgg :: GroupedDataFrame -> SomeExpr -> Maybe AggPlan
 planAgg gdf (SomeExpr (expr :: Expr a)) = case expr of
-    Agg (FoldAgg tag _ _) (Col name) -> foldPlan tag name
-    Agg (MergeAgg tag _ _ _ _) (Col name) -> mergePlan tag name
-    Agg (CollectAgg tag _) (Col name) -> collectPlan tag name
+    Agg (FoldAgg (Builtin tag) _ _) (Col name) -> foldPlan tag name
+    Agg (MergeAgg (Builtin tag) _ _ _ _) (Col name) -> mergePlan tag name
+    Agg (CollectAgg (Builtin tag) _) (Col name) -> collectPlan tag name
     Binary
         op
-        (Agg (FoldAgg lt Nothing _) (Col a))
-        (Agg (FoldAgg rt Nothing _) (Col b)) ->
-            if binaryName op == "sub" && lt == "maximum" && rt == "minimum"
-                then requireBoth a b (PlanMaxMinusMin a b)
-                else Nothing
+        (Agg (FoldAgg (Builtin AMaximum) Nothing _) (Col a))
+        (Agg (FoldAgg (Builtin AMinimum) Nothing _) (Col b))
+            | Builtin BSub <- binaryId op -> requireBoth a b (PlanMaxMinusMin a b)
     _ -> Nothing
   where
     foldPlan tag name = case tag of
-        "sum" -> require name (PlanScatter RSum name)
-        "minimum" -> require name (PlanScatter RMin name)
-        "maximum" -> require name (PlanScatter RMax name)
+        ASum -> require name (PlanScatter RSum name)
+        AMinimum -> require name (PlanScatter RMin name)
+        AMaximum -> require name (PlanScatter RMax name)
         _ -> Nothing
     mergePlan tag name = case tag of
-        "mean" -> outputType @Double >> require name (PlanScatter RMean name)
-        "count" -> outputType @Int >> require name (PlanScatter RCount name)
-        "countDistinct" -> do
+        AMean -> outputType @Double >> require name (PlanScatter RMean name)
+        ACount -> outputType @Int >> require name (PlanScatter RCount name)
+        ACountDistinct -> do
             outputType @Int
             c <- getColumn name (fullDataframe gdf)
             if distinctSupported c then Just (PlanCountDistinct name) else Nothing
@@ -79,9 +82,9 @@
         Just Refl -> Just ()
         Nothing -> Nothing
     collectPlan tag name = case tag of
-        "stddev" -> require name (PlanScatter RStd name)
-        "variance" -> require name (PlanScatter RVar name)
-        "median" -> require name (PlanMedian name)
+        AStddev -> require name (PlanScatter RStd name)
+        AVariance -> require name (PlanScatter RVar name)
+        AMedian -> require name (PlanMedian name)
         _ -> Nothing
     require name plan = colUnboxedNumeric name >> Just plan
     requireBoth a b plan = colUnboxedNumeric a >> colUnboxedNumeric b >> Just plan
@@ -162,8 +165,8 @@
 -- | Tag a single named aggregation with its moment role, or reject the group.
 classify :: M.Map T.Text SomeExpr -> (T.Text, SomeExpr) -> Maybe (T.Text, Role)
 classify exprs (name, SomeExpr expr) = case expr of
-    Agg (MergeAgg "count" _ _ _ _) _ -> Just (name, RoleN)
-    Agg (FoldAgg "sum" _ _) arg -> (\t -> (name, termRole t)) <$> resolveTerm exprs (SomeExpr arg)
+    Agg (MergeAgg (Builtin ACount) _ _ _ _) _ -> Just (name, RoleN)
+    Agg (FoldAgg (Builtin ASum) _ _) arg -> (\t -> (name, termRole t)) <$> resolveTerm exprs (SomeExpr arg)
     _ -> Nothing
 
 termRole :: Term -> Role
@@ -179,9 +182,10 @@
         Col nm -> case M.lookup nm exprs of
             Just ue -> go (fuel - 1) ue
             Nothing -> Just (Lin nm)
-        Unary _ inner -> go (fuel - 1) (SomeExpr inner)
+        Unary op inner
+            | Builtin UToDouble <- unaryId op -> go (fuel - 1) (SomeExpr inner)
         Binary op l r
-            | binaryName op == "mult" && binaryCommutative op -> do
+            | Builtin BMult <- binaryId op -> do
                 Lin a <- go (fuel - 1) (SomeExpr l)
                 Lin b <- go (fuel - 1) (SomeExpr r)
                 Just (sortProd a b)