fun mask (guess, answer) =
  let
    fun mask2 (m, i, [], answer) = m
      | mask2 (m, i, letter :: rest, answer) =
          mask2 ((m * 3
          + (if sub(answer, i) = letter
               then 2
             else if isSubstring(str letter) answer
               then 1
             else 0)), i + 1, rest, answer)
  in
    mask2 (0, 0, explode guess, answer)
  end;

fun maskToString m =
  let
    fun maskToString2 (m, s, 0) = s
      | maskToString2 (m, s, k) =
        maskToString2 (m div 3,
          List.nth(["b", "y", "g"], m mod 3) ^ s,
          k - 1)
  in
    maskToString2 (m, "", 5)
  end;

val words = from w in file.wordle.words yield w.word;

fun maskCount (guess, remainingWords) =
  from w in remainingWords
    group m = mask (guess, w) compute c = count
    compute count;

fun bestGuesses words =
  from w in words,
    maskCount = maskCount (w, words)
    order maskCount desc;

fun remaining (words, []) = words
  | remaining (words, (guess, m) :: rest) =
      from w in (remaining (words, rest))
      where maskToString (mask (guess, w)) = m;

Functional programming – values, types, operators

1 + 2;
> val it = 3 : int
"Hello, " ^ "world!";
> val it = "Hello, world!" : string
val integers = [1, 2, 3, 4, 5, 6, 7, 8];
> val integers = [1,2,3,4,5,6,7,8] : int list
fun filter f [] = []
  | filter f (first :: rest) =
      if (f first)
        then first :: (filter f rest)
        else filter f rest;
> val filter = fn : ('a -> bool) -> 'a list -> 'a list
filter (fn i => i mod 2 = 0) integers;
> val it = [2,4,6,8] : int list

val union = fn : 'a list * 'a list -> 'a list
val except = fn : 'a list * 'a list -> 'a list
val intersect = fn : 'a list * 'a list -> 'a list
val filter = fn : ('a -> bool) -> 'a list -> 'a list
val map = fn : ('a -> 'b) -> 'a list -> 'b list
val join = fn
  : 'a list * 'b list * ('a * 'b -> bool)
    -> ('a * 'b) list

from e in db.emps
  where e.deptno = 10
  yield {e.name, pay = e.sal + e.comm}

-- SQL
SELECT item, COUNT(*) AS c,
  SUM(sales) AS total
FROM ProduceSales
WHERE item != 'bananas'
  AND category IN ('fruit', 'nut')
GROUP BY item
ORDER BY item DESC;

item      c total
====== ==== =====
apples    2     9

-- GoogleSQL pipe syntax
FROM ProduceSales
|> WHERE item != 'bananas'
    AND category IN ('fruit', 'nut')
|> AGGREGATE COUNT(*) AS c, SUM(sales) AS total
   GROUP BY item
|> ORDER BY item DESC;

from p in produceSales
  where p.item != "bananas"
    andalso p.category elem ["fruit", "nut"]
  group p.item compute c = count,
    total = sum of p.sales
  order item desc

SELECT e.ename, e.sal
FROM emps AS e
WHERE e.deptno = 10
AND e.sal > (SELECT MAX(e2.sal)
             FROM emps AS e2
             WHERE e2.deptno = 20
             AND e2.job = 'PROGRAMMER')

from e in scott.emps
  where e.deptno = 10
    andalso sal >
    (from e2 in scott.emp
      where e2.deptno = 20
        andalso e2.job = "PROGRAMMER"
      compute max 0 of e2.sal)
  yield e.ename

from e in scott.emp
  where e.deptno = 10
    andalso
    (forall e2 in scott.emp
      where e2.deptno = 20
        andalso e2.job = "PROGRAMMER"
      require e.sal > e2.sal)
  yield e.ename

datatype personnel_id =
    EMPLOYEE of int
  | CONTRACTOR of {ssid: string, agency: string};

type member = {name: string, deptno: int, id: personnel_id};

val members = [
  {name = "Smith", deptno = 10, id = EMPLOYEE 100},
  {name = "Jones", deptno = 20,
   id = CONTRACTOR {ssid = "xxx-xx-xxxx", agency = "Cheap & cheerful"}];

val departments = scott.depts;

val primes = [2, 3, 5, 7, 11];

val bands = [["john", "paul", "george", "ringo"], ["simon", "garfunkel"]];

from i in [1, 2],
    j in ["a", "b"],
    k in [3, 4, 5, 6]
  where i + k < 6;
> {i: int, j: string, k: int} list

from dept in [10, 30],
    e in scott.emps
  where e.deptno = dept
  yield e.ename;
> string bag

from dept in [10, 30],
    e in scott.emps
  where e.deptno = dept
  order e.sal desc
  take 3
  yield {e.deptno, e.ename};
> {deptno: int, ename: string} list

-- Delete employees who earn more than 1,000.
DELETE FROM scott.emps
WHERE sal > 1000;

-- Add one employee.
INSERT INTO scott.emps (empno, deptno, ename, job, sal)
VALUES (100, 20, 'HYDE', 'ANALYST', 1150);

-- Double the salary of all managers.
UPDATE scott.emps
SET sal = sal * 2
WHERE job = 'MANAGER';

-- Commit.
COMMIT;

(* Delete employees who earn more than 1,000. *)
delete e in scott.emps
  where e.sal > 1000;
(* Add one employee. *)
insert scott.emps
  [{empno = 100, deptno = 20, ename = "HYDE",
    job = "ANALYST", sal = 1150}];
(* Double the salary of all managers. *)
update e in scott.emps
  where e.job = "MANAGER"
  assign (e, {e with sal = e.sal * 2});
(* Commit. *)
commit;

(* Delete employees who earn more than 1,000. *)
val emps2 =
  from e in scott.emps
    where not (e.sal > 1000);
(* Add one employee. *)
val emps3 = emps2 union
  [{empno = 100, deptno = 20, ename = "HYDE", job = "ANALYST",
    sal = 1150}];
(* Double the salary of all managers. *)
val emps4 =
  from e in emps3
    yield if e.job = "MANAGER"
      then {e with sal = e.sal * 2}
      else e;
(* Commit. *)
commit {scott with emps = emps4};

(* New and removed employees. *)
val empsAdded = emps4 except scott.emps;
val empsRemoved = scott.emps except emps4;
(* Compute the updated summary table. *)
val summary2 =
  from s in scott.summary
    union
      (from e in empsAdded
        yield {e.deptno, c = 1, sum_sal = e.sal}
    union
      (from e in empsRemoved
        yield {e.deptno, c = ~1, sum_sal = ~sum_sal})
    group s.deptno compute c = sum of c, sum_sal = sum of sum_sal
    where c != 0);
(* Commit. *)
commit {scott with summary = summary2};

(* Morel "forwards" relation *)
(* Relation defined using algebra. *)
fun clerks () =
  from e in emps
    where e.job = "CLERK";
(* Query uses regular iteration. *)
from e in clerks,
    d in depts
  where d.deptno = e.deptno
  andalso d.loc = "DALLAS"
  yield e.name;
val it =
  ["SMITH", "ADAMS"] : string list;

(* Morel "backwards" relation *)
(* Relation defined using a predicate. *)
fun isClerk e =
  e.job = "CLERK";
(* Query uses a mixture of constrained
   and regular iteration. *)
from e,
    d in depts
  where isClerk(e)
    andalso d.deptno = e.deptno
    andalso d.loc = "DALLAS"
  yield e.name;
val it =
  ["SMITH", "ADAMS"] : string list;

datatype 'a option = NONE | SOME of 'a;
SOME 1;

- from e in scott.emp
=   where e.deptno = 10
=   andalso (forall e2 in scott.emp
=     where e2.job = "PROGRAMMER"
=     require e.sal > e2.sal)
=   yield {e.ename, e.sal};
val it =
  [{ename="CLARK",sal=2450.0},{ename="KING",sal=5000.0},
   {ename="MILLER",sal=1300.0}] : {ename:string, sal:real} list

(* Initial database value and type (schema). *)
val scott1 = db
  : {emps: {name: string, empno: int, deptno: int,
            hiredate: string} bag,
    depts: {deptno: int, name: string} bag};
(* Shim that makes a v1 database look like v2. *)
fun scott2on1shim scott1 =
  {emps =
    fn () => from e in scott1.emps
      yield {e with hiredate = Date.fromString(e.hiredate)},
   depts = fn () => scott1.depts};
(* Shim that makes v3 database look like v1. *)
fun scott1on3shim scott3 =
  {emps =
    fn () => from e in scott3.emps
      yield {e with hiredate = Date.toString(e.hiredate)
               removing rating},
   depts = fn () => scott3.depts};
(* An application writes its queries & views against version 2;
   shims make it work on any actual version. *)
val scott = scott2;
fun recentHires () =
  from e in scott.e
    where e.hiredate > Date.subtract(Date.now(), 100);

An agent can turn out code at much faster than any human developer, but like a human, it has trouble grasping the big picture. A 100K-line system is too large for a single context, but if it is divided into smaller pieces, the agent can write and maintain them one at a time.

This is where the human developer — the architect — can help. I define architecture as a strategy for organizing complexity, but usually involve divides the system into components with clean interfaces. An agent (or human) working on one component needs only a brief, accurate picture of the others — not their source.

Components take many forms, such as microservices, APIs, and libraries. A single project can contain components, provided that care is taken to clearly define each component’s boundary. This post focuses on tools, which are the simplest kind of component.

Unix tools

Unix tools are effective components because they have a simple interface contract (the man page), are easy to invoke (from the shell), and compose with other tools (in pipelines and scripts).

If you have comments, please reply on Bluesky @julianhyde.bsky.social or Twitter:

You can now write queries in the Soufflé dialect of Datalog and execute them using Morel’s usual runtime.

This demonstrates that Morel now supports both query paradigms — Datalog’s relational calculus and Morel’s native relational algebra — and you can freely switch between them. But what are these paradigms, and why does it matter?

The two paradigms

The two paradigms originate in set theory, and continue through the relational model into modern query languages.

Set theory provides two ways to define a set: the intensional method defines the set by its properties (for example, “red cars” is the set of all cars whose color is red), and the extensional method creates the set by performing operations on existing sets (intersect the set of all cars with the set of all red objects).

The relational model for databases provides two ways to specify a query which mirror intensional and extensional set definitions. In relational calculus, one specifies the logical properties of the tuples to retrieve from the input relations; in relational algebra, one specifies the input relations and a sequence of operations (intersect, join, filter, project) to apply to them. Codd’s Theorem proves that these languages have equivalent expressive power.

Query languages are generally based on one of those paradigms. SQL is largely based on algebra (although its EXISTS keyword shows the influence of calculus). Datalog is based on calculus. Functional programming languages (including Morel) are in the algebra camp; they provide relational operators via higher-order functions like map, filter and reduce, and sometimes provide syntactic sugar like list-comprehensions.

If the languages are equivalent, why does it matter? The languages have different strengths.

Algebra’s strengths:

• Algebra naturally extends to bags and lists (collections with ordering and/or duplicate values), while calculus only works on sets;
• Aggregate functions are a more natural extension to algebra than calculus;
• Mainstream programming languages are functional or procedural, so there is lower impedance mismatch embedding a query in a program or writing a user-defined function to be called from a query;
• Developers familiar with mainstream programming languages find the calculus paradigm difficult to learn.

Calculus (epitomized by Datalog) excels at graph and deductive queries, such as queries that iterate until they reach a fixed point. As we shall see, it is just easier to write recursive queries if they return a boolean than if they return a complex data type like a set of tuples.

For simple fixed-point queries such as computing the transitive closure of a relation, the algebra query returns a set that is the union of the points that are one step away, two steps away, and so forth. In calculus, the value is boolean: whether there is a path from one point to another.

For more complex fixed-point queries, the algebra programmer must define a data type with a semilattice structure. Consider, for example, a query to find all pairs of nodes connected by no more than five steps. In algebra, the data type is now a set of (source, destination, distance) triples combined by taking the minimum distance. In calculus, the data type remains boolean: the result of the function has_path_within(source, destination, distance). The boolean function is easier to write, and easier for the query planner to understand.

Until now, if a programmer had to solve a problem with mixed workload, they would need to switch languages. Because of a new feature called predicate inversion, Morel now supports both paradigms.

The Datalog interface

The following program, in the Soufflé dialect of Datalog, computes the transitive closure of an edge relation.

.decl edge(x:number, y:number)
.decl path(x:number, y:number)
edge(1,2).
edge(2,3).
path(X,Y) :- edge(X,Y).
path(X,Z) :- path(X,Y), edge(Y,Z).
.output path

In a graph with nodes 1, 2 and 3, the edge relation defines edges from 1 → 2 and 2 → 3. The derived path relation says that there is a path between two nodes if (a) there is an edge, or (b) there is an edge to an intermediate node and a path from that intermediate node to the destination node. From the edges {1 → 2, 2 → 3} it deduces the paths {1 → 2, 2 → 3, 1 → 3}.

You can now run the following program from Morel’s shell:

The program is passed (as a string literal) as an argument to the Datalog.execute function, and the Soufflé symbol and number types in the .decl directive have been mapped to Morel string and int types, but is otherwise unchanged.

(Adding a Datalog structure, with functions execute, translate and validate, seemed preferable to writing a whole Datalog shell and testing framework. Facts and rules have the same syntax as Soufflé, as does the .output directive. The .input directive, not shown in this example, has a new optional filePath argument.)

Translating Datalog to Morel

The translation makes concrete the equivalence that Codd’s Theorem promises: each Datalog construct has a direct counterpart in Morel.

One way to support Datalog would have been to implement a Datalog engine, but this would have been a major task and would not have benefited the rest of Morel. Instead, we have extended the Morel language with Datalog-like constructs; this has made the Morel language more powerful, and made Datalog translation straightforward.

The Datalog-to-Morel translator has a structure that will be familiar to anyone who has implemented a compiler that translates a high-level language to a lower-level language. Three steps are executed in succession:

1. The parser converts a Datalog string to a parse tree.
2. The validator makes sure that the program is valid (that rules are safe, grounded and stratified) and deduces its type.
3. The translator generates a Morel program that is equivalent to the Datalog program.

Parsing and validation follow standard patterns, but let’s look at the translation algorithm in a little more detail. Here is the translation to Morel of the earlier Datalog program:

You’ll notice that the Datalog and Morel programs have the same structure. Datalog rules without a body (such as edge(1,2) and edge(2,3)) are gathered into a list of tuples (edge_facts).

Each rule becomes a boolean function. If there are several comma-separated predicates in a rule’s body, they are combined using andalso. If there are several rules of the same name, their conditions are combined using orelse. Invocations of a rule become function calls, which, like rules, may be recursive.

The body of the rule path(X,Z) :- path(X,Y), edge(Y,Z) has a variable, Y, that does not occur in the head. It is translated to exists v0.

A Datalog program may have several .output directives. The Morel program returns a single value, a record with one field for each directive. This program has one directive, .output path, so the record has a single field named path that is a list of {x:int, y:int} records.

How Morel does it

The magic lies not in the Datalog-to-Morel converter but in the Morel language itself. Over the last few months, we have added to Morel a capability called predicate inversion, the ability to deduce a set from a boolean expression.

At the heart of the generated Morel program is a query: from x, y where path (x, y). It differs from a regular query in that the variables x and y are unbounded. (In a conventional query, every variable is bounded, meaning it iterates over a collection, as do d and e in from d in departments, e in employees.)

In principle, an unbounded variable iterates over every possible value of its data type. This is fine for “small” data types like boolean, char, and enum Color { RED | GREEN | BLUE }, but problematic for “large” data types like int and {b: boolean, i: int} and infinite data types like string and int list.

Morel allows unbounded variables in a program as long as there is a predicate like where x > 0 andalso x < 10 or where e elem employees that connects it with a finite set. Invertible predicates provide a way to generate the values of the variable. In Datalog parlance, they ensure that the variable is grounded.

Morel’s predicate inversion algorithm recognizes various predicate patterns, including boolean functions that check collection membership (like edge) or compute transitive closure (like path).

Mixing styles

The net result is that predicate inversion allows you to freely mix Datalog-style queries (defined by boolean expressions and functions) with the relational algebra-style queries (defined by from, exists, join and set operations).

The following query is in a hybrid style.

The edge and reachable functions define graph reachability in a Datalog style, using recursion and boolean return values. The from query is in the algebra style, but uses predicate inversion to generate all values of the unbounded target variable for which reachable (source, target) is true. Predicate inversion provides the junction between the two styles.

Conclusion

Morel now unifies the calculus and algebra styles of writing queries. The new Datalog interface showcases this capability, but you can also use the calculus style in Morel programs, where you can freely mix it with the algebra style and functional programming.

Notice that we have made no claims about the efficiency of the implementation. Our goal was to increase the expressive power of the language, and we have achieved that goal. Morel’s internal representation is algebraic — using relational operators, other operators provided by functions, and iteration to a fixed point — and from this point we can apply conventional query-optimization techniques.

To keep things simple, we have not discussed evaluation models. Datalog uses forward chaining (bottom-up evaluation) while boolean functions give the impression that backwards chaining (top-down evaluation) is being used. For most queries both approaches are valid, and the planner would ideally consider both strategies along with optimizations such as join re-ordering, magic sets, semi-naïve evaluation, and materialized views. But there are queries where the evaluation model matters (say, they would terminate under one model but not another), and for these cases it is important that we define Morel’s operational semantics.

The predicate inversion algorithm needs to evolve and mature. It has been tested over a wide array of queries, but there are still cases where it fails to invert a predicate, or fails to remove a condition that has been fully satisfied by a generator. (We hope to write more about predicate inversion, generators, and subsuming predicates, in a future article.)

Please download Morel and give it a try! (Morel has both Java and Rust versions, but Datalog and predicate inversion require the Java version for now.)

If you have comments, please reply on Bluesky @julianhyde.bsky.social or Twitter:

This article has been updated.

The following table lists the year that various programming languages introduced lambda syntax (not always the year in which the language was born). If a language introduced an alternate syntax at a different date, I have noted the year of introduction.

(Please let me know if there are mistakes in syntax or year of introduction. Claude was my research assistant. I omitted languages in the same family with the same syntax, e.g. Lisp-Scheme-Racket, OCaml-F#. Did I miss any early, major languages?)

Here is the original tweet:

This article has been updated.

Footnotes

The Morel language has an existing implementation in Java (Morel Java version 0.7 was released in June and 0.8 is coming soon) but this is the beginning of a brand-new Rust runtime.

What’s in release 0.2.0

This release focuses on Morel’s underpinnings as a functional programming language. It can parse any program, and execute simple programs that consist of expressions, function declarations, and lambdas (closures).

Here’s a quick example showing what works today. First, use cargo to build Morel and start a shell:

$ cargo run
morel-rust version 0.2.0 (rust version 1.90.0)
-

Next, you can enter some commands:

This demonstrates core functional programming: recursion, higher-order functions, and composition. Support for programs that contain queries—the from, exists, and forall keywords—and user-defined types will come later.

A caveat: this is pre-alpha software. Expect bugs, crashes, and minimal error handling. We’ve focused on getting the foundations right—the Hindley-Milner type deduction algorithm, an evaluation environment that handles recursive functions and closures—rather than polish. If you’d like to contribute—fixing bugs, adding features, or improving documentation—please join us!

Why Rust?

Why create a Rust runtime for Morel when there is already a Java runtime? Rust brings significant advantages for data processing workloads.

Rust processes in-memory data at exceptional speed, with zero-cost abstractions and no garbage collection pauses. It integrates naturally with modern data infrastructure: Apache DataFusion for query execution, Arrow for columnar processing, and Parquet for efficient storage. Memory safety comes without runtime overhead, and the resulting binaries are ideal for cloud-native deployments.

Having multiple runtimes also underscores a key design principle: when writing a Morel program, you don’t need to think about implementation details. Your choice of runtime is separate from your choice of language. Choose Java for its ecosystem and JVM integration, or Rust for performance and modern infrastructure. Programs are portable across both.

But the most important “runtime” is wherever your data already lives—Iceberg tables on object storage, Kafka topics, or SQL engines like Snowflake, BigQuery, or Postgres. That’s why query planning, federation, and SQL dialect translation are central to Morel’s design. The compiler can push computation to the data, regardless of which Morel implementation you’re using.

Morel is a language, not a framework

Morel is a complete language, not a framework. When you have a data problem, you can solve it entirely in Morel—no jumping between languages, no glue code, and no framework boundaries to cross.

With a framework, you’re constantly context-switching: Python for orchestration, SQL for queries, Java for business logic, and Spark for transformations. Morel lets you express the entire solution in one language, bringing the benefits of functional programming—type safety, composability, and refactoring—to data engineering.

Choose your runtime, keep your code

Because Morel is a language with multiple implementations, your Morel programs are portable across runtimes. Write your code once, and run it on either Java or Rust—whichever fits your deployment needs. Users shouldn’t notice—and don’t need to care—which implementation they’re running.

One reason that Morel Rust has developed quickly is that we can run Morel Java’s test scripts unchanged. We are gradually enabling tests as functionality comes online, proving portability in practice.

Learn more

To find out more about Morel, read about its goals and basic language, and find a full definition of the language in the query reference and the language reference.

If you have comments, please reply on Bluesky @julianhyde.bsky.social or Twitter:

This article has been updated.

Apache Arrow, Apache DataFusion, Apache Iceberg, Apache Parquet, and Apache Kafka are trademarks of the Apache Software Foundation.

Why expressions?

In release 0.6, Morel’s query syntax (simplified a little) looked like this:

query → from scan [ , scan ... ] [ step ... ]

step → distinct
    | except [ distinct ] exp [ , exp ... ]
    | group groupKey [ , groupKey ... ] [ compute agg [ , agg ... ] ]
    | intersect [ distinct ] exp [ , exp ... ]
    | join scan [ , scan ... ]
    | order orderItem [ , orderItem ... ]
    | skip exp
    | take exp
    | union [ distinct ] exp [ , exp ... ]
    | where exp
    | yield exp

scan → pat in exp [ on exp ]

orderItem → exp [ desc ]

groupKey → [ id = ] exp

agg → [ id = ] exp [ of exp ]

Almost everything is an expression. The argument to the yield step is an expression (whereas SQL’s SELECT has a list of expressions with optional AS aliases); the scan in a from query or join step is over an expression (which, unlike SQL, is not necessarily a query); the arguments to the where, skip, take, union, intersect, and union steps are also expressions.

(The groupKey and agg items in group and compute have some way to go, and we will be looking at those for Morel 0.8, but at least the aggregate function (before of) may be a (function-valued) expression.)

Making everything an expression pays dividends. Queries can return a collection of any value, not just records. You can easily join a collection to a set of nested records (say an order to its nested order-lines). If you need a custom aggregate function, you can roll your own. And each of these expressions can be made into function arguments, so that you can parameterize your query.

From Morel 0.7 onwards, syntax of the order step is simpler:

step → ...
  | order exp

The argument is now just an expression, and the orderItem concept has disappeared.

Let’s look at how we got here. What was wrong with the previous syntax, which alternatives did we consider for the new syntax, and what changes were necessary in order to make it possible?

The order step

In the previous syntax, the argument of the order step was a comma-separated list of order-items, each of which is an expression with an optional desc keyword.

One problem is the commas. In the expression

it is not immediately clear whether j is a second argument for the call to the function foo or the second item in the order clause.

Another problem was the fact that the order clause could not be empty. The ordered and unordered collections feature introduced an unorder step to convert a list to a bag, and we need the opposite of that, a trivial sort whose key has the same value for every element.

We can’t just get rid of the desc keyword and covert the list to a singleton. Real queries require complex sorting behaviors like composite keys, descending keys, and nulls-first or nulls-last specifications. So, how can we put all that complexity in a single expression?

One approach is to do what many programming languages do, and use a comparator function. Let’s explore this approach.

Comparator functions

In Standard ML, a comparator function is any function that takes a pair of arguments of the same type and returns a value of the order enum (LESS, EQUAL, GREATER). Its type is alpha * alpha -> order.

For int, I can write a simple function:

In fact, most data types have a built-in compare function:

For more complex orderings, I can write a comparator that combines other comparators. For example, this function compares a list of string * real pairs, the string first, then the real descending:

If we were to add comparators to Morel, we could add order using syntax like this:

(The comparator expression in this query is basically an inline version of the compareStringRealPair function, but working on emp records rather than string * real pairs.)

But this is much longer than the equivalent in SQL. Comparator functions are clearly powerful, but they fail the “make simple things simple” test – forcing developers to write complex code for common sorting patterns.

Let’s look instead at value-based sorting, which is simpler, but provides most of the flexibility of comparator functions.

Structured values for complex orderings

The idea behind value-based sorting is that any values of the same type can be compared, and that the Morel system generates comparison logic for any type. If you require a complex sorting behavior, you can construct an expression with a complex type.

Previously, if you wanted a composite ordering, with one of the keys descending, you would write something like this:

As of Morel 0.7, you can write the same query using a single expression:

Note that:

• For a composite ordering, we use a tuple type. Morel compares the values lexicographically.
• For a descending ordering, we wrap the value in the descending data type using its DESC constructor. Morel compares the values in the usual way, then reverses the direction.

Sorting is defined for all other data types, including tuples, records, sum-types such as Option and Descending, lists, bags, and any combination thereof.

Morel’s compiler has two tricks to make this powerful and efficient.

First, Morel is effectively generating a comparator function at compile time based on the type of the order expression. This makes value-based sorting as powerful as comparator functions, but with less code for the user to write.

(The change included a new library function, Relational.compare, that allows you to compare any two values of the same type, even if you are not performing a sort. This is a somewhat strange function, because it takes the type as an implicit argument, then drives its behavior by introspecting that type.)

Second, the order clause uses a form of lazy evaluation. If the query

created a tuple (e.job, DESC(e.sal)) for every element, we would worry about the impact on performance, but those tuples are never constructed. Morel operates on the employee records e directly, and the performance is the same as if we had specified the ordering using a list of order-items or a comparator function.

Benefits of sorting on expressions

Now the order step takes an expression, what is now possible that wasn’t before?

We can pass the expression as an argument to a function, like this:

We can also achieve the trivial sort required to convert a bag to a list. You can sort by any constant value, such as the integer 0 or the Option constructor NONE, but the norm would be to sort by the empty tuple ():

Note that result is a list, even though scott.emps (a relational database table) is a bag. The elements are in arbitrary order (because any order is consistent with the empty sort key) but in converting the collection to a list the arbitrary order has become frozen and repeatable.

Future work

Several challenges remain to be addressed.

NULLS FIRST and NULLS LAST

Real-world data sets often contain null values, and at various times you wish to sort nulls low (as if they were zero or negative infinity) or high (as if they were positive infinity). Morel uses the option type rather than NULL to represent optional values, but the same requirement exists.

SQL has NULLS FIRST and NULLS LAST keywords to control how nulls are sorted, but Morel does not have an equivalent syntax.

Currently, the behavior is the same as SQL’s NULLS FIRST. This happens because Morel sorts datatype values based on the declaration order of their constructors. The option type is declared as:

Since NONE appears before SOME in this declaration, the NONE value sorts lower than all SOME values:

We haven’t yet figured out how to express the equivalent of NULLS LAST. One idea is to add a noneLast datatype

and use it in a query like this:

When we use NONE_LAST and DESC together in a query

the NONE value appears first. It’s what we asked for, but not what we expected if we were expecting DESC and NONE_LAST to commute.

Until we figure out something intuitive, we won’t have a solution for NULLS LAST yet.

Comparator functions

Under the “make hard things possible” principle, we might still want to support comparator functions at some point. The syntax could be as follows:

step → ...
  | order exp
  | order using comparator

Is value-based sorting strictly less powerful than comparator functions? It’s an interesting theoretical question, and I honestly don’t know. A comparator function can be an arbitrarily complex piece of code — but perhaps it is always possible to create a value that matches the structure of the code.

Aggregation syntax

The syntax for group and compute steps is still not an expression. For Morel 0.8 and beyond, we will be looking at several improvements.

First, making the group-key and compute-items an expression, with field aliasing provided via record syntax, as in the current yield step.

Second, allowing complex compute expressions with expressions both inside and outside the aggregate function, as in the SQL expression “1 + AVG(sal * 2)”. This will mean the of keyword, which is currently part of the agg syntax, will be transitioned to a new keyword that is part of the expression syntax, possibly over.

Third, further explore the relationship between the argument to an aggregate function and a query. Noting that SQL aggregate function syntax by now includes most relational operators (FILTER, DISTINCT, WITHIN DISTINCT, ORDER BY) consider making the argument (the over keyword just mentioned) a kind of query expression.

Conclusion

Making sorting expression-based represents more than just a syntax change – it exemplifies Morel’s commitment to principled language design. By eliminating the special-case syntax for order, we’ve resolved parsing ambiguities and enabled new forms of query composition.

In the next few releases, we shall continue to evolve Morel to make it more uniform and composable. The result, we hope, will be a query language that feels both familiar to SQL users and naturally functional to developers who think in terms of higher-order functions and data transformation pipelines.

To find out more about Morel, read about its goals and basic language, peruse the query reference or language reference, or download it from GitHub and give it a try.

If you have comments, please reply on Bluesky @julianhyde.bsky.social or Twitter:

This article has been updated.

This release has actually been under development for a long time. Ordered and unordered collections and queries, which are the centerpiece of this release, required major changes to the type inference algorithm, not to mention a new data type (bag), query step (unorder), and expression (ordinal). The type inference changes have been under development for six months (during which time there were two other Morel releases), and were so extensive that we got function overloading practically for free.

There are other changes to query syntax: sorting on expressions, atomic yield steps, and set operators in pipelines.

Morel aims to be a solid implementation of Standard ML and good general-purpose programming language, in addition to being a revolutionary query language, which means gradually completing our implementation of Standard ML’s Basis Library. This release we have completed the String and Char structures.

Let’s explore the key features. For complete details, see the official release notes.

1. Ordered and unordered collections and queries

The biggest change in 0.7.0 is the introduction of ordered and unordered collections and queries. Previously, every query was over a list type, whose elements were ordered and duplicates were allowed.

But saying that every collection and query is over a list type is a white lie. Consider this query:

The collection scott.emps maps to the EMP table in the scott database, and Morel’s goal is to push as much of the processing as possible to where the data resides. In this case, Morel can generate the SQL query

SELECT ENAME
FROM SCOTT.EMP
WHERE SAL > 1000.0;

SQL makes no guarantees about the order of results. If you execute the query twice, a DBMS is free to return the results in a different order each time. So Morel is being dishonest if it says that result is a list.

Could we redefine list so that its iteration order is undefined? Yes, but then we would be short-changing queries such as

which do have a defined order.

The fact is – even though the relational model tells us it ain’t so – some data sets are ordered, and some are unordered. Adding distinct bag and list types, relational operators that can work on both, and relational operators to convert between them, was the way to go.

The features that we implemented are described in the article “Ordered and unordered data”.

2. Function overloading

In Standard ML, and in Morel until recently, a name could only have one binding. Functions are values, and therefore inhabit the same namespace as regular values. If I declare x to be an int value

and then later try to declare x to be a function

then the previous declaration of x is no longer accessible.

To create overloaded functions, we need declare that an identifier is special; we do this using the new over keyword:

Now we can define several instances of f:

All must be functions, because the overloads are resolved based on the type of the first argument.

Calls to f will be resolved based on the types of the arguments:

3. Sorting on expressions

There are only a few places in Morel syntax where you do not use an expression, and the order step used to be one of them. Previously, order was followed by a list of “order items”, each an expression optionally followed by desc. The items were separated by commas, and the list could not be empty.

The commas were a problem. In the expression

it is not clear whether j is a second argument for the call to the function foo or the second item in the order clause.

Another problem was the fact that the order clause could not be empty. The ordered/unordered collections feature introduced an unorder step to convert a list to a bag, and we need the opposite of that, a trivial sort whose key has the same value for every element.

The answer was to make the argument to order an expression. A composite sort specification is now a tuple, still separated by commas, but now enclosed in parentheses. If a sort key is descending, you now wrap it in the Descending data type by preceding it with the DESC. Thus:

You can now sort by any data type, including tuples, records, sum-types such as Option and Descending, lists, bags, and any combination thereof.

To achieve the trivial sort, you can sort by any constant value, such as the integer 0 or the Option constructor NONE, but conventionally you would sort by the empty tuple ():

The key thing is that the result is a list. The elements are in arbitrary order (because any order is consistent with the empty sort key) but in converting the collection to a list the arbitrary order has become frozen and repeatable.

4. Atomic yield steps

At any step in a Morel query, there are generally several named fields you can use to reference parts of the current row. For example, the where step in the following query refers to both fields, i and j.

But there is one circumstance where a step does not produce any named fields: a yield whose expression is not a record, what we call an “atomic yield”. Here is an example:

That query is valid, but suppose we wished to sort or filter the results. If we added an order or where step it would have no way to refer to the current row. We allowed atomic yields because we needed queries with non-record elements, but we made a rule that the atomic yield had to be the last step.

That restriction was becoming more of a burden, and the final straw was ordered/unordered queries, which often end in order or unorder. So we decided to fix the problem.

We added a new expression, current, that refers to the current element. (It is only available in query steps, but you can use it inside a sub-expression or sub-query.) If the value is atomic, current is that value; if there are named fields, current is a record consisting of those fields. (In the previous example, current would be equivalent to {i, j}.)

If a yield is atomic but the expression has a clear name, as in yield i or yield e.deptno, you can also use that name. (The expression is still considered atomic, and the result of the query will be a collection of that type, not a collection of records.)

Here are some examples of current in action.

5. Set operators in pipelines

The set operators (union, intersect and except) were previously available via functions but now have dedicated steps in the query pipeline.

The steps have slightly different semantics for ordered and unordered collections, and have an optional distinct keyword to eliminate duplicates.

For example, here is a query that finds all employees in departments 10 and 20, but excludes those who are managers or clerks:

If you have ever wondered about the semantics of intersect and except with duplicates, wonder no more! INTERSECT ALL, EXCEPT ALL, and the arithmetic of fractions explains everything using a fun example.

6. String and Char structures

Morel now includes complete String and Char structures following the Standard ML Basis Library specification.

This gives you comprehensive text manipulation capabilities:

These structures provide everything you need for serious text processing, from basic operations like substring extraction to advanced features like tokenization and character classification.

7. Breaking changes

This release includes some breaking changes to be aware of.

Database schema updates

The scott sample database now uses pluralized table names, mapping the emps value maps to the EMP table, and depts to the DEPT table.

This change aligns with the modern programming convention that collections have plural names.

Type-based orderings

The previous order syntax no longer works.

You should convert a following desc to preceding DESC:

and put parentheses around composite orderings:

Conclusion

Release 0.7.0 represents a major evolution in Morel’s capabilities. Extensions to the query language, type system, and standard library make Morel a good solution for a wide range of data processing tasks, from simple queries to complex data transformations.

As always, you can get started with Morel by visiting GitHub. For more background, read about its goals and basic language, and find a full definition of the language in the query reference and the language reference.

If you have comments, please reply on Bluesky @julianhyde.bsky.social or Twitter:

This article has been updated.

I’m not talking about sorted data. If you sort a collection, applying a comparator function to its elements, then you have no more information than you had before.

No, the integer list

and the string list

depend on the order of their elements for their meaning.

But of course, some data is unordered, for good reason. A relational database would be foolish to guarantee that if you write rows into a table in a particular order, they will be read back in the same order. Such a guarantee would seriously limit the database’s scalability.

This post is about how we allow ordered and unordered data to coexist in Morel.

We achieved this with a collection of new features, including adding a bag type, the ordered relational operators, the ordinal keyword, and a new unorder step. All of these features will appear shortly in Morel release 0.7.

List and bag types

As a functional query language, Morel spans the worlds of database and functional programming.

Databases’ fundamental type, the relation, is an unordered collection of records. (Though curiously, modern SQL allows columns to contain “nested tables”, which can be either of the ordered ARRAY type or the unordered MULTISET type.)

Functional programming languages’ fundamental type is the list, an ordered type. Functional programs are often defined by structural induction on lists. For example, the function

inductively defines that a list of numbers is “all-positive” if it is empty, or if its first element is positive and the rest of the list is “all-positive”. This kind of inductive definition requires a firm distinction between the first element of a list and the rest of the list, a distinction that is not present in an unordered collection.

So, Morel needs to support both ordered and unordered collections.

Earlier versions of Morel papered over the difference. All collections had type list, even the unordered collections backed by database tables. Morel’s relational operators produced results in deterministic order if you applied them to in-memory collections using the in-process interpreter, but order was not guaranteed when Morel converted the query to SQL for execution in a DBMS.

To fix the problem, the first step was to add a bag type. (Bag is a synonym for multiset, implying a given element may occur more than once, but iteration order is not defined.) bag is the unordered counterpart to the ordered list type, and has similar operations.

Order-dependent operations from the list type, such as hd and drop, are defined for bag instances, but they are not guaranteed to return the same result every time you call them.

Collections backed by database tables now have type bag:

(You may notice that scott.depts collection, backed by the DEPT table of the SCOTT JDBC data source, has changed its name as well as its type. It used to be called scott.dept. Morel collection names should be plural and lower-case, and improvements to the name mapping system make it easier to derive proper collection names.)

Next, we provide relational operators to convert between list and bag.

Converting between ordered and unordered

Now that queries can reference list and bag collections, we need operators to convert from one to the other. To do this, we use the existing order step, and add an unorder step and an ordinal expression.

In previous versions of Morel, the order step converted a list to a list with a different ordering; now its input can be a list or a bag:

If the sort key does not create a total ordering, the results will be nondeterministic but still a list. For example, we can sort integers so that even numbers occur before odd numbers

or convert a bag to a list in arbitrary order.

To go the opposite direction, the new unorder step converts a list to a bag:

(You are also free to apply unorder to a bag; it will have no effect.)

As we said above, a bag contains less information than its corresponding list. If you plan to convert the bag to a list at a later stage, you need to store the ordering in an extra field. The new ordinal expression lets us do this:

The ordinal expression can be used in an expression in a step whose input is ordered (except the steps whose expressions are evaluated before the query starts: except, intersect, skip, take, and union). ordinal evaluates to 0 for the first element, 1 for the next element, and so on. But as we shall see, the optimizer avoids evaluating ordinal if it can.

Here is a query that computes the salary rank of each employee, then returns only the poorly-paid clerks.

The main reason to apply order and unorder in a query is to control the target collection type. But there is a more subtle reason which relates to performance. The ordered and unordered versions of the relational operators may produce the same results (modulo ordering) but ordered execution may be less efficient (such as running with reduced parallelism). If a query contains an order or unorder, the order of the input to that step is irrelevant, and the optimizer can use a more efficient execution plan.

This, by the way, is why the specification of the order step does not guarantee stability. If order was stable, the optimizer would have to use ordered execution of upstream steps if the sort key is not exhaustive.

If you want order to be stable, you can add ordinal to the trailing edge of the sort key:

Materializing ordinal as a 1-based, contiguous sequence of integers is expensive because it forces sequential execution, and the optimizer will avoid this if it can. In this case, because ordinal is used for sorting but is not returned, the optimizer downgrades ordinal to a virtual expression. The plan might use an ordered implementation of the where and yield steps followed by a stable sort, or it might replace ordinal with the previous sort key (DESC e.sal).

Ordered relational operators

We need to define the semantics of the relational operators over all types of collection.

Part of the job has been done already:

• The relational model defines the semantics of operators over sets (unordered collections without duplicates).
• The SQL standard specifies the relational operators over tables (unordered collections with duplicates).
• Previous versions of Morel defined semantics for (and implemented) relational operators over multisets (unordered collections with duplicates). While the collection type was at the time called list, we were actually defining the current bag type. Unlike SQL, elements need not be records.

What remains is to define the semantics of queries over lists (ordered collections with duplicates), and for hybrid queries that combine lists and multisets. (We define hybrid semantics in the next section.)

Because a query consists of a sequence of steps, each corresponding to a relational operator, we define the semantics of each step over input that is a list:

• The first step in a query – from pat in exp, forall pat in exp, or exists pat in exp – returns elements in the same order that they are emitted from exp.
• join pat in exp [ on condition ] for each element from its input evaluates exp, then, in order of those elements, emits a record consisting of fields of the two elements, skipping records where condition is false.
• If a from, forall, exists or join step has more than one scan, each subsequent scan behaves as if it were a separate join step.
• yield exp preserves order.
• where condition preserves order, dropping rows for which condition is false.
• skip count and take count preserve order (respectively dropping the first count rows, or taking the first count rows).
• distinct preserves order, emitting only the first occurrence of each element.
• group groupKey1, ..., groupKeyg [ compute agg1, ..., agga ] preserves order, emitting groups in the order that the first element in the group was seen; each aggregate function aggi is invoked with a list of the input elements that belong to that group, in arrival order.
• compute agg1, ..., agga behaves as a group step where all input elements are in the same group.
• union [ distinct ] exp1, ..., expn outputs the elements of the input in order, followed by the elements of each exp_i argument in order (just like the UNIX cat command). If distinct is specified, outputs only the first occurrence of each element.
• intersect [ distinct ] exp1, ..., expn outputs the elements of the input in order, provided that every exp_i argument contains at least the number of occurrences of this element so far. If distinct is specified, outputs only the first occurrence of each element.
• except [ distinct ] exp1, ..., expn outputs the elements of the input in order, provided that the number of occurrences of that element so far is less than the number of occurrences of that element in all the exp_i arguments. If distinct is specified, outputs only the first occurrence of each element.
• require condition (which can only occur in a forall query) has the same behavior as the unordered case.
• order and unorder, as discussed earlier, have the same semantics as in the unordered case.

The rules for from and join produce the same familiar ordering as a nested “for” loop in a language such as C, Python or Java:

The rules for union, intersect and except are rather subtle, and are best illustrated by example:

Hybrid relational operators

We have specified the behavior of queries where input collections are all lists or all bags. But what if a query has a mix of list and bag inputs?

The mixing can occur if the first step of the query (from, exists, or forall) has more than one scan, or in steps that introduce another collection (join, union, intersect, or except). In all cases, unordered wins: if any input is a bag, the step becomes unordered, and unordered semantics apply from then on.

For example, if we join a list of department numbers (ordered) to a table of employees (unordered), selecting only the clerks and managers, the result is a bag (unordered):

Type inference challenges

This feature was challenging to implement because it required major changes to Morel’s type inference algorithm. (We mention this only in the spirit of sharing war-stories, and for the interest of those who understand the internal workings of Morel’s compiler. Hopefully, the changes to type-inference algorithm will be invisible to the casual user.)

The problem is evident in a program such as

While resolving the type of function f and its embedded query, the types of the arguments xs and ys have not yet been determined. Morel’s previous type inference algorithm allowed us to say “xs and ys must be lists with the same element type” or “xs and ys must be bags with the same element type”. It was based on Hindley-Milner’s Algorithm W and unification, which basically means finding an assignment of logical variables so that two trees are structurally identical.

But the type inference rules for queries with a mixture of ordered and unordered collections require conditions that contain the word ‘or’. For example, resolving the intersect expression above requires that we say “we can allow xs and ys to both be bags, or both be lists, or one to be a bag and the other a list, but they must have same element type”. Furthermore, we need to derive the result type, saying “the result of the query is a list if both arguments are lists, otherwise a bag, with the same element type as the arguments”.

We needed a system where we can place a number of constraints on type variables, and then solve for those constraints. The new type inference algorithm extends Hindley-Milner with constraints, using the approach described in “A Second Look at Overloading” by Odersky, Wadler & Wehr (1995). As the title of that paper suggests, we have added a kind of overloading to Morel; it is as if the intersect operator now has four forms:

• intersect: α bag * α bag → α bag
• intersect: α bag * α list → α bag
• intersect: α list * α bag → α bag
• intersect: α list * α list → α list

(and similar overloads for the other relational operators) and the type inference algorithm solves the constraints to land on one valid assignment of types.

The algorithm took several months of hard work to implement, but the results are pleasing. Morel retains the key benefits of a Hindley-Milner type system: strong static typing, runtime efficiency, and type inference without the need for type annotations.

Like any other major change in architecture, constraint-based type inference will take a while to mature; [MOREL-270] and [MOREL-271] describe some of the remaining issues.

Conclusion

The ability to combine ordered and unordered data sets, and process both using relational operators, is a major new feature in Morel. It allows Morel to handle, with equal ease, data from files and relational databases, and data that is generated programmatically.

This feature will be available in Morel release 0.7.

To find out more about Morel, read about its goals and basic language, peruse the query reference or language reference, or download it from GitHub and give it a try.

If you have comments, please reply on Bluesky @julianhyde.bsky.social or Twitter:

SQL’s set operators (UNION, INTERSECT, and EXCEPT) have set and multiset variants. The multiset variants retain duplicates and use the ALL keyword; the set variants discard duplicates, and you can use the optional DISTINCT keyword if you want to be explicit.

Morel has just added union, intersect and except query steps, achieving parity with both Standard SQL and GoogleSQL’s pipe syntax. (This post is about multiset mode, which retains duplicate values; to use the set mode, which discards duplicates and is far more common, use the distinct keyword, for example intersect distinct.)

Using these steps, we can compute GCD and LCM. The queries are even more concise because Morel queries over integer values do not require column names.

Adding fractions

Remember how – probably in middle school – you learned how to add two fractions, and to reduce a fraction to its lowest terms?

Suppose you need to add 5/36 and 7/120. First, find the least common multiple (LCM) of their denominators (36 and 120).

Next, convert each fraction to an equivalent fraction with the LCM (360) as the denominator.

• For 5/36: Multiply the numerator and denominator by 10 (since 36 * 10 = 360).
• For 7/120: Multiply the numerator and denominator by 3 (since 120 * 3 = 360).

Last, add the fractions.

  5      7        5 * 10    7 * 3        50      21       71
---- + -----  =  ------- + -------  =  ----- + -----  =  -----
 36     120      36 * 10   120 * 3      360     360       360

Computing GCD and LCM

To compute the GCD of two numbers, you start by finding their prime factors. Prime factors can be repeated, so these are multisets, not sets. Let’s find the GCD of 36 and 120.

• 36 is 2² * 3², so has factors [2, 2, 3, 3]
• 120 is 2³ * 3 * 5, so has factors [2, 2, 2, 3, 5]

Where there are factors in common, we take the lower repetition count. Taking the minimum count for each common factor – two 2s, one 3, and no 5s – the GCD is therefore 2² * 3, which is 12.

The crucial step of the algorithm is to combine two multisets and take the minimum repetition count; that is exactly what intersect does.

The LCM is similar, but takes the higher repetition count. This can be achieved by taking the union of both factor multisets, then subtracting their intersection. Here’s why: The union gives us all factors from both numbers, but it adds the counts together. Since we want the maximum count (not the sum), we subtract the intersection, which contains the overlapping factors we double-counted. The LCM is therefore 2³ * 3² * 5, which is 360.

Using Morel to compute LCM and GCD

To convert this algorithm to code, we will need three things:

• a factorize function splits the numbers into multisets of prime factors;
• the intersect step combines the multisets;
• a product function converts the multisets back to a number.

Here are the factorize and product functions.

Here’s how they work:

So, we can compute GCD like this:

The last step uses compute because product fulfills Morel’s only criterion to be an aggregate function: its argument is a collection of values. (At least one SQL dialect agrees with us, and has a PRODUCT aggregate function.)

LCM can be computed from GCD:

But, as we discussed above, it can also be computed directly using union, except and intersect:

Let’s test them:

Conclusion

The intersect, except, and union steps neatly solve the problem of computing GCD and LCM because they handle repeated factors in exactly the way we need.

These steps will be available shortly in Morel release 0.7.

If you have comments, please reply on Bluesky @julianhyde.bsky.social or Twitter:

This article has been updated.