Add FlatArray<'_, T> (pgcentralfoundation#2207)

Introduce a new Rust type for what ArrayTypes "really are": FlatArray.
As an unsized type, it will primarily be used as `&FlatArray<'_, T>` or
similar in actual extension code. The long-term plan is for FlatArray to
phase out `pgrx::datum::Array<'_, T>` entirely, as it has several
advantages over its predecessor:

- It uses `Element`, a subset of `BorrowDatum`, to power its simplicity
in iteration, making it more easily verifiable and maintainable.
- It is the unsized type itself, allowing composing it with various
Rusty pointer types like `&`, `&mut`, or `PBox`.
- It is no longer endangered by performance cliffs or hidden allocations
depending on the type of `T`.
- Because of these previous traits, it is possible to soundly and safely
allocate a fresh `FlatArray`, mutate it, and return that one allocation,
instead of allocating an intermediate Vec in Rust memory to mutate!

It also has an associated cost in that it no longer supports any `T`
that requires complex unboxing of items like `datum::Array<'a, String>`.
You must instead use `array.iter().map(closure)` to accomplish that.
This itself summarizes the difference: `datum::Array` is effectively an
iteration protocol, but `array::FlatArray` is an actual data structure.

This addresses the implied feature request in, and thus supercedes,
pgcentralfoundation#2086.

Author: workingjubilee

pgrx-tests/src/tests/array_borrowed.rs

@@ -218,6 +218,7 @@ mod tests {
 
     #[pg_test(expected = "attempt to add with overflow")]
     fn test_sum_array_i32_overflow() -> Result<Option<i64>, pgrx::spi::Error> {
+        // Note that this test is calling a builtin, array_agg
         Spi::get_one::<i64>(
             "SELECT sum_array(a) FROM (SELECT array_agg(s) a FROM generate_series(1, 1000000) s) x;",
         )

pgrx-tests/src/tests/array_tests.rs

@@ -11,6 +11,7 @@ mod aggregate_tests;
 mod anyarray_tests;
 mod anyelement_tests;
 mod anynumeric_tests;
+mod array_borrowed;
 mod array_tests;
 mod attributes_tests;
 mod bgworker_tests;

pgrx-tests/src/tests/mod.rs

@@ -0,0 +1,11 @@
+use pgrx::array::FlatArray;
+use pgrx::prelude::*;
+
+// We could support this behavior but it would require allocating a fresh instance,
+// which is better represented by using PBox or similar for the input array.
+#[pg_extern]
+fn something<'a>(_arr: &mut FlatArray<'a, i32>) {
+    todo!()
+}
+
+fn main() {}

pgrx-tests/tests/compile-fail/array-as-non-mutable-args.rs

@@ -0,0 +1,47 @@
+error[E0277]: cannot generate SQL schema for this function
+ --> tests/compile-fail/array-as-non-mutable-args.rs:7:1
+  |
+6 | #[pg_extern]
+  | ------------ in this procedural macro expansion
+7 | fn something<'a>(_arr: &mut FlatArray<'a, i32>) {
+  | ^^ some types in for<'b, 'a> fn(&'b mut FlatArray<'a, i32>) cannot be translated to SQL
+  |
+  = help: the trait `FunctionMetadata<_>` is not implemented for `for<'b, 'a> fn(&'b mut FlatArray<'a, i32>)`
+  = note: this error originates in the attribute macro `pg_extern` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error[E0599]: `&mut FlatArray<'_, i32>` has no representation in SQL
+ --> tests/compile-fail/array-as-non-mutable-args.rs:6:1
+  |
+6 | #[pg_extern]
+  | ^^^^^^^^^^^^ non-SQL type
+  |
+  = note: the following trait bounds were not satisfied:
+          `&mut FlatArray<'_, i32>: SqlTranslatable`
+          which is required by `&&mut FlatArray<'_, i32>: SqlTranslatable`
+  = note: this error originates in the attribute macro `pg_extern` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error[E0277]: `&mut FlatArray<'_, i32>` cannot be passed into a Postgres function as a Datum
+ --> tests/compile-fail/array-as-non-mutable-args.rs:7:18
+  |
+6 | #[pg_extern]
+  | ------------ in this procedural macro expansion
+7 | fn something<'a>(_arr: &mut FlatArray<'a, i32>) {
+  |                  ^^^^ the trait `ArgAbi<'_>` is not implemented for `&mut FlatArray<'_, i32>`
+  |
+  = help: the following other types implement trait `ArgAbi<'fcx>`:
+            &'fcx T
+            &'fcx [u8]
+            &'fcx str
+            &MemCx<'fcx>
+            *mut FunctionCallInfoBaseData
+            AnyArray
+            AnyElement
+            AnyNumeric
+          and $N others
+  = note: `ArgAbi<'_>` is implemented for `&FlatArray<'_, i32>`, but not for `&mut FlatArray<'_, i32>`
+note: required by a bound in `pgrx::callconv::Args::<'a, 'fcx>::next_arg_unchecked`
+ --> $WORKSPACE/pgrx/src/callconv.rs
+  |
+  |     pub unsafe fn next_arg_unchecked<T: ArgAbi<'fcx>>(&mut self) -> Option<T> {
+  |                                         ^^^^^^^^^^^^ required by this bound in `Args::<'a, 'fcx>::next_arg_unchecked`
+  = note: this error originates in the attribute macro `pg_extern` (in Nightly builds, run with -Z macro-backtrace for more info)

pgrx-tests/tests/compile-fail/array-as-non-mutable-args.stderr

@@ -0,0 +1,11 @@
+use pgrx::array::FlatArray;
+use pgrx::memcx::MemCx;
+use pgrx::palloc::PBox;
+use pgrx::prelude::*;
+
+#[pg_extern]
+fn something<'a>(_mcx: &MemCx<'a>) -> PBox<'a, FlatArray<'a, FlatArray<'a, i32>>> {
+    todo!()
+}
+
+fn main() {}

pgrx-tests/tests/compile-fail/no-arrays-of-arrays.rs

@@ -0,0 +1,38 @@
+error[E0277]: cannot generate SQL schema for this function
+ --> tests/compile-fail/no-arrays-of-arrays.rs:7:1
+  |
+6 | #[pg_extern]
+  | ------------ in this procedural macro expansion
+7 | fn something<'a>(_mcx: &MemCx<'a>) -> PBox<'a, FlatArray<'a, FlatArray<'a, i32>>> {
+  | ^^ some types in for<'b, 'a> fn(&'b MemCx<'a>) -> PBox<'a, FlatArray<'a, FlatArray<'a, i32>>> cannot be translated to SQL
+  |
+  = help: the trait `FunctionMetadata<_>` is not implemented for `for<'b, 'a> fn(&'b MemCx<'a>) -> PBox<'a, FlatArray<'a, FlatArray<'a, i32>>>`
+  = note: this error originates in the attribute macro `pg_extern` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error[E0599]: `PBox<'_, FlatArray<'_, FlatArray<'_, i32>>>` has no representation in SQL
+ --> tests/compile-fail/no-arrays-of-arrays.rs:6:1
+  |
+6 | #[pg_extern]
+  | ^^^^^^^^^^^^ non-SQL type
+  |
+ ::: $WORKSPACE/pgrx/src/palloc/pbox.rs
+  |
+  | pub struct PBox<'mcx, T: ?Sized> {
+  | -------------------------------- doesn't satisfy `_: SqlTranslatable`
+  |
+ ::: $WORKSPACE/pgrx/src/array/flat_array.rs
+  |
+  | pub struct FlatArray<'mcx, T: ?Sized> {
+  | ------------------------------------- doesn't satisfy `FlatArray<'_, FlatArray<'_, i32>>: SqlTranslatable`
+  |
+note: if you're trying to build a new `PBox<'_, FlatArray<'_, FlatArray<'_, i32>>>`, consider using `PBox::<'mcx, T>::from_raw_in` which returns `PBox<'_, _>`
+ --> $WORKSPACE/pgrx/src/palloc/pbox.rs
+  |
+  |     pub unsafe fn from_raw_in(ptr: NonNull<T>, _cx: &MemCx<'mcx>) -> PBox<'mcx, T> {
+  |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  = note: the following trait bounds were not satisfied:
+          `FlatArray<'_, FlatArray<'_, i32>>: SqlTranslatable`
+          which is required by `PBox<'_, FlatArray<'_, FlatArray<'_, i32>>>: SqlTranslatable`
+          `PBox<'_, FlatArray<'_, FlatArray<'_, i32>>>: SqlTranslatable`
+          which is required by `&PBox<'_, FlatArray<'_, FlatArray<'_, i32>>>: SqlTranslatable`
+  = note: this error originates in the attribute macro `pg_extern` (in Nightly builds, run with -Z macro-backtrace for more info)

pgrx-tests/tests/compile-fail/no-arrays-of-arrays.stderr

@@ -0,0 +1,12 @@
+use pgrx::array::FlatArray;
+use pgrx::palloc::PBox;
+use pgrx::prelude::*;
+
+// We can support this behavior, it just requires allocating a fresh instance,
+// which is something we want to approach systematically for similar datums.
+#[pg_extern]
+fn something<'a>(_arr: PBox<'a, FlatArray<'a, i32>>) {
+    todo!()
+}
+
+fn main() {}

pgrx-tests/tests/todo/array-as-pbox.rs

@@ -0,0 +1,24 @@
+error[E0277]: `PBox<'_, FlatArray<'_, i32>>` cannot be passed into a Postgres function as a Datum
+ --> tests/todo/array-as-pbox.rs:8:18
+  |
+7 | #[pg_extern]
+  | ------------ in this procedural macro expansion
+8 | fn something<'a>(_arr: PBox<'a, FlatArray<'a, i32>>) {
+  |                  ^^^^ the trait `ArgAbi<'_>` is not implemented for `PBox<'_, FlatArray<'_, i32>>`
+  |
+  = help: the following other types implement trait `ArgAbi<'fcx>`:
+            &'fcx T
+            &'fcx [u8]
+            &'fcx str
+            &MemCx<'fcx>
+            *mut FunctionCallInfoBaseData
+            AnyArray
+            AnyElement
+            AnyNumeric
+          and $N others
+note: required by a bound in `pgrx::callconv::Args::<'a, 'fcx>::next_arg_unchecked`
+ --> $WORKSPACE/pgrx/src/callconv.rs
+  |
+  |     pub unsafe fn next_arg_unchecked<T: ArgAbi<'fcx>>(&mut self) -> Option<T> {
+  |                                         ^^^^^^^^^^^^ required by this bound in `Args::<'a, 'fcx>::next_arg_unchecked`
+  = note: this error originates in the attribute macro `pg_extern` (in Nightly builds, run with -Z macro-backtrace for more info)

pgrx-tests/tests/todo/array-as-pbox.stderr

@@ -8,18 +8,31 @@
 //LICENSE
 //LICENSE Use of this source code is governed by the MIT license that can be found in the LICENSE file.
 #![allow(clippy::precedence)]
-use crate::datum::Array;
+#![allow(unused)]
+#![deny(unsafe_op_in_unsafe_fn)]
+use crate::datum::{Array, BorrowDatum, Datum};
+use crate::layout::{Align, Layout};
+use crate::memcx::MemCx;
+use crate::nullable::Nullable;
+use crate::palloc::PBox;
+use crate::pgrx_sql_entity_graph::metadata::{
+    ArgumentError, Returns, ReturnsError, SqlMapping, SqlTranslatable,
+};
 use crate::toast::{Toast, Toasty};
 use crate::{layout, pg_sys, varlena};
-use bitvec::ptr::{self as bitptr, BitPtr, BitPtrError, Mut};
-use bitvec::slice::BitSlice;
+use bitvec::ptr::{self as bitptr, BitPtr, BitPtrError, Const, Mut};
+use bitvec::slice::{self as bitslice, BitSlice};
+use core::iter::{ExactSizeIterator, FusedIterator};
+use core::marker::PhantomData;
 use core::ptr::{self, NonNull};
-use core::slice;
+use core::{ffi, mem, slice};
 
 mod element;
+mod flat_array;
 mod port;
 
 pub use element::Element;
+pub use flat_array::{ArrayAllocError, FlatArray};
 
 /**
 An aligned, dereferenceable `NonNull<ArrayType>` with low-level accessors.
@@ -165,8 +178,7 @@ impl RawArray {
         }
     }
 
-    /**
-    A slice of the dimensions.
+    /** A slice describing the array's dimensions.
 
     Oxidized form of [ARR_DIMS(ArrayType*)][ARR_DIMS].
     The length will be within 0..=[pg_sys::MAXDIM].
@@ -189,6 +201,7 @@ impl RawArray {
     }
 
     /// The flattened length of the array over every single element.
+    ///
     /// Includes all items, even the ones that might be null.
     ///
     /// # Panics
@@ -229,8 +242,7 @@ impl RawArray {
         // This field is an "int32" in Postgres
     }
 
-    /**
-    Equivalent to [ARR_HASNULL(ArrayType*)][ARR_HASNULL].
+    /** Equivalent to [ARR_HASNULL(ArrayType*)][ARR_HASNULL].
 
     Note this means that it only asserts that there MIGHT be a null
 
@@ -250,16 +262,26 @@ impl RawArray {
     }
 
     #[inline]
-    fn nulls_bitptr(&mut self) -> Option<BitPtr<Mut, u8>> {
+    fn nulls_bitptr(&self) -> Option<BitPtr<Const, u8>> {
+        let nulls_ptr = unsafe { port::ARR_NULLBITMAP(self.ptr.as_ptr()) }.cast_const();
+        match BitPtr::try_from(nulls_ptr) {
+            Ok(ptr) => Some(ptr),
+            Err(BitPtrError::Null(_)) => None,
+            Err(BitPtrError::Misaligned(_)) => unreachable!(),
+        }
+    }
+
+    #[inline]
+    fn nulls_mut_bitptr(&mut self) -> Option<BitPtr<Mut, u8>> {
+        let nulls_ptr = unsafe { port::ARR_NULLBITMAP(self.ptr.as_ptr()) };
         match BitPtr::try_from(self.nulls_mut_ptr()) {
             Ok(ptr) => Some(ptr),
             Err(BitPtrError::Null(_)) => None,
-            Err(BitPtrError::Misaligned(_)) => unreachable!("impossible to misalign *mut u8"),
+            Err(BitPtrError::Misaligned(_)) => unreachable!(),
         }
     }
 
-    /**
-    Oxidized form of [ARR_NULLBITMAP(ArrayType*)][ARR_NULLBITMAP]
+    /** Oxidized form of [ARR_NULLBITMAP(ArrayType*)][ARR_NULLBITMAP]
 
     If this returns None, the array cannot have nulls.
     If this returns Some, it points to the bitslice that marks nulls in this array.
@@ -278,8 +300,8 @@ impl RawArray {
         NonNull::new(ptr::slice_from_raw_parts_mut(self.nulls_mut_ptr(), len))
     }
 
-    /**
-    The [bitvec] equivalent of [RawArray::nulls].
+    /** The [bitvec] equivalent of [RawArray::nulls].
+
     If this returns `None`, the array cannot have nulls.
     If this returns `Some`, it points to the bitslice that marks nulls in this array.
 
@@ -291,23 +313,21 @@ impl RawArray {
     [ARR_NULLBITMAP]: <https://git.postgresql.org/gitweb/?p=postgresql.git;a=blob;f=src/include/utils/array.h;h=4ae6c3be2f8b57afa38c19af2779f67c782e4efc;hb=278273ccbad27a8834dfdf11895da9cd91de4114#l293>
     */
     pub fn nulls_bitslice(&mut self) -> Option<NonNull<BitSlice<u8>>> {
-        NonNull::new(bitptr::bitslice_from_raw_parts_mut(self.nulls_bitptr()?, self.len()))
+        NonNull::new(bitptr::bitslice_from_raw_parts_mut(self.nulls_mut_bitptr()?, self.len()))
     }
 
-    /**
-    Checks the array for any NULL values by assuming it is a proper varlena array,
+    /** Checks the array for any NULL values
 
     # Safety
-
     * This requires every index is valid to read or correctly marked as null.
+
     */
     pub unsafe fn any_nulls(&self) -> bool {
         // SAFETY: Caller asserted safety conditions.
         unsafe { pg_sys::array_contains_nulls(self.ptr.as_ptr()) }
     }
 
-    /**
-    Oxidized form of [ARR_DATA_PTR(ArrayType*)][ARR_DATA_PTR]
+    /** Oxidized form of [ARR_DATA_PTR(ArrayType*)][ARR_DATA_PTR]
 
     # Safety
 
@@ -395,7 +415,7 @@ impl Toasty for RawArray {
 ///
 /// This allows for it to be copied from a Rust slice to a Postgres array,
 /// and obtain a Rust slice from a Postgres array if it contains no nulls.
-pub unsafe trait Scalar: Sized + Copy {
+pub unsafe trait Scalar: Sized + Copy + Element {
     const OID: pg_sys::Oid;
 }