WIP: .move_into() and better .try_append_array()

FEAT: automatic layout fix in append_to_array (doc not yet updated)

FEAT: New method .move_into() for moving all array elements

TEST: Add tests for .move_into()

We use a DropCounter to check duplication/drops of elements rigorously.
The DropCounter code is taken from rayon collect tests, where I wrote
it.

Author: bluss

src/impl_owned_array.rs

@@ -1,9 +1,12 @@
 
 use alloc::vec::Vec;
+use std::mem::MaybeUninit;
 
 use crate::imp_prelude::*;
+
 use crate::dimension;
 use crate::error::{ErrorKind, ShapeError};
+use crate::iterators::Baseiter;
 use crate::OwnedRepr;
 use crate::Zip;
 
@@ -137,6 +140,166 @@ impl<A> Array<A, Ix2> {
 impl<A, D> Array<A, D>
     where D: Dimension
 {
+    /// Move all elements from self into `new_array`, which must be of the same shape but
+    /// can have a different memory layout. The destination is overwritten completely.
+    ///
+    /// ***Panics*** if the shapes don't agree.
+    pub fn move_into(mut self, new_array: &mut Array<MaybeUninit<A>, D>) {
+        unsafe {
+            // Safety: copy_to_nonoverlapping cannot panic
+            // Move all reachable elements
+            Zip::from(self.raw_view_mut()).and(new_array)
+                .for_each(|src, dst| {
+                    src.copy_to_nonoverlapping(dst.as_mut_ptr(), 1);
+                });
+            // Drop all unreachable elements
+            self.drop_unreachable_elements();
+        }
+    }
+
+    /// This drops all "unreachable" elements in the data storage of self.
+    ///
+    /// That means those elements that are not visible in the slicing of the array.
+    /// *Reachable elements are assumed to already have been moved from.*
+    ///
+    /// # Safety
+    ///
+    /// This is a panic critical section since `self` is already moved-from.
+    fn drop_unreachable_elements(mut self) -> OwnedRepr<A> {
+        let self_len = self.len();
+
+        // "deconstruct" self; the owned repr releases ownership of all elements and we
+        // and carry on with raw view methods
+        let data_len = self.data.len();
+
+        let has_unreachable_elements = self_len != data_len;
+        if !has_unreachable_elements || std::mem::size_of::<A>() == 0 {
+            unsafe {
+                self.data.set_len(0);
+            }
+            self.data
+        } else {
+            self.drop_unreachable_elements_slow()
+        }
+    }
+
+    #[inline(never)]
+    #[cold]
+    fn drop_unreachable_elements_slow(mut self) -> OwnedRepr<A> {
+        // "deconstruct" self; the owned repr releases ownership of all elements and we
+        // and carry on with raw view methods
+        let self_len = self.len();
+        let data_len = self.data.len();
+        let data_ptr = self.data.as_nonnull_mut().as_ptr();
+
+        let mut self_;
+
+        unsafe {
+            // Safety: self.data releases ownership of the elements
+            self_ = self.raw_view_mut();
+            self.data.set_len(0);
+        }
+
+
+        // uninvert axes where needed, so that stride > 0
+        for i in 0..self_.ndim() {
+            if self_.stride_of(Axis(i)) < 0 {
+                self_.invert_axis(Axis(i));
+            }
+        }
+
+        // Sort axes to standard order, Axis(0) has biggest stride and Axis(n - 1) least stride
+        // Note that self_ has holes, so self_ is not C-contiguous
+        sort_axes_in_default_order(&mut self_);
+
+        unsafe {
+            let array_memory_head_ptr = self_.ptr.as_ptr();
+            let data_end_ptr = data_ptr.add(data_len);
+            debug_assert!(data_ptr <= array_memory_head_ptr);
+            debug_assert!(array_memory_head_ptr <= data_end_ptr);
+
+            // iter is a raw pointer iterator traversing self_ in its standard order
+            let mut iter = Baseiter::new(self_.ptr.as_ptr(), self_.dim, self_.strides);
+            let mut dropped_elements = 0;
+
+            // The idea is simply this: the iterator will yield the elements of self_ in
+            // increasing address order.
+            //
+            // The pointers produced by the iterator are those that we *do not* touch.
+            // The pointers *not mentioned* by the iterator are those we have to drop.
+            //
+            // We have to drop elements in the range from `data_ptr` until (not including)
+            // `data_end_ptr`, except those that are produced by `iter`.
+            let mut last_ptr = data_ptr;
+
+            while let Some(elem_ptr) = iter.next() {
+                // The interval from last_ptr up until (not including) elem_ptr
+                // should now be dropped. This interval may be empty, then we just skip this loop.
+                while last_ptr != elem_ptr {
+                    debug_assert!(last_ptr < data_end_ptr);
+                    std::ptr::drop_in_place(last_ptr as *mut A);
+                    last_ptr = last_ptr.add(1);
+                    dropped_elements += 1;
+                }
+                // Next interval will continue one past the current element
+                last_ptr = elem_ptr.add(1);
+            }
+
+            while last_ptr < data_end_ptr {
+                std::ptr::drop_in_place(last_ptr as *mut A);
+                last_ptr = last_ptr.add(1);
+                dropped_elements += 1;
+            }
+
+            assert_eq!(data_len, dropped_elements + self_len,
+                       "Internal error: inconsistency in move_into");
+        }
+        self.data
+    }
+
+    /// Create an empty array with dimension `ndim` and all zeros shape
+    ///
+    /// ***Panics*** if ndim is 0 or D is zero-dim
+    pub(crate) fn empty(ndim: usize) -> Array<A, D> {
+        assert_ne!(ndim, 0);
+        assert_ne!(D::NDIM, Some(0));
+        unsafe {
+            // Safety: all elements (zero elements) are initialized
+            Array::uninit(D::zeros(ndim)).assume_init()
+        }
+    }
+
+    /// Create new_array with the right layout for appending to `growing_axis`
+    #[inline(never)]
+    fn change_to_contiguous_layout(&mut self, growing_axis: Axis) {
+        let ndim = self.ndim();
+        let mut dim = self.raw_dim();
+
+        // The array will be created with 0 (C) or ndim-1 (F) as the biggest stride
+        // axis. Rearrange the shape so that `growing_axis` is the biggest stride axis
+        // afterwards.
+        let prefer_f_layout = growing_axis == Axis(ndim - 1);
+        if !prefer_f_layout {
+            dim.slice_mut().swap(0, growing_axis.index());
+        }
+        let mut new_array = Self::uninit(dim.set_f(prefer_f_layout));
+        if !prefer_f_layout {
+            new_array.swap_axes(0, growing_axis.index());
+        }
+
+        // self -> old_self.
+        // dummy array -> self.
+        // old_self elements are moved -> new_array.
+        let old_self = std::mem::replace(self, Self::empty(ndim));
+        old_self.move_into(&mut new_array);
+
+        // new_array -> self.
+        unsafe {
+            *self = new_array.assume_init();
+        }
+    }
+
+
     /// Append an array to the array
     ///
     /// The axis-to-append-to `axis` must be the array's "growing axis" for this operation
@@ -217,19 +380,27 @@ impl<A, D> Array<A, D>
         }
 
         let self_is_empty = self.is_empty();
+        let mut incompatible_layout = false;
 
         // array must be empty or have `axis` as the outermost (longest stride) axis
         if !self_is_empty && current_axis_len > 1 {
             // `axis` must be max stride axis or equal to its stride
             let max_stride_axis = self.axes().max_by_key(|ax| ax.stride).unwrap();
             if max_stride_axis.axis != axis && max_stride_axis.stride > self.stride_of(axis) {
-                return Err(ShapeError::from_kind(ErrorKind::IncompatibleLayout));
+                incompatible_layout = true;
             }
         }
 
         // array must be be "full" (have no exterior holes)
         if self.len() != self.data.len() {
-            return Err(ShapeError::from_kind(ErrorKind::IncompatibleLayout));
+            incompatible_layout = true;
+        }
+
+        if incompatible_layout {
+            self.change_to_contiguous_layout(axis);
+            // safety-check parameters after remodeling
+            debug_assert_eq!(self_is_empty, self.is_empty());
+            debug_assert_eq!(current_axis_len, self.len_of(axis));
         }
 
         let strides = if self_is_empty {
@@ -313,7 +484,7 @@ impl<A, D> Array<A, D>
                         array.invert_axis(Axis(i));
                     }
                 }
-                sort_axes_to_standard_order(&mut tail_view, &mut array);
+                sort_axes_to_standard_order_tandem(&mut tail_view, &mut array);
             } 
             Zip::from(tail_view).and(array)
                 .debug_assert_c_order()
@@ -336,7 +507,21 @@ impl<A, D> Array<A, D>
     }
 }
 
-fn sort_axes_to_standard_order<S, S2, D>(a: &mut ArrayBase<S, D>, b: &mut ArrayBase<S2, D>)
+/// Sort axes to standard order, i.e Axis(0) has biggest stride and Axis(n - 1) least stride
+///
+/// The axes should have stride >= 0 before calling this method.
+fn sort_axes_in_default_order<S, D>(a: &mut ArrayBase<S, D>)
+where
+    S: RawData,
+    D: Dimension,
+{
+    if a.ndim() <= 1 {
+        return;
+    }
+    sort_axes1_impl(&mut a.dim, &mut a.strides);
+}
+
+fn sort_axes_to_standard_order_tandem<S, S2, D>(a: &mut ArrayBase<S, D>, b: &mut ArrayBase<S2, D>)
 where
     S: RawData,
     S2: RawData,
@@ -350,6 +535,32 @@ where
                   a.shape(), a.strides());
 }
 
+fn sort_axes1_impl<D>(adim: &mut D, astrides: &mut D)
+where
+    D: Dimension,
+{
+    debug_assert!(adim.ndim() > 1);
+    debug_assert_eq!(adim.ndim(), astrides.ndim());
+    // bubble sort axes
+    let mut changed = true;
+    while changed {
+        changed = false;
+        for i in 0..adim.ndim() - 1 {
+            let axis_i = i;
+            let next_axis = i + 1;
+
+            // make sure higher stride axes sort before.
+            debug_assert!(astrides.slice()[axis_i] as isize >= 0);
+            if (astrides.slice()[axis_i] as isize) < astrides.slice()[next_axis] as isize {
+                changed = true;
+                adim.slice_mut().swap(axis_i, next_axis);
+                astrides.slice_mut().swap(axis_i, next_axis);
+            }
+        }
+    }
+}
+
+
 fn sort_axes_impl<D>(adim: &mut D, astrides: &mut D, bdim: &mut D, bstrides: &mut D)
 where
     D: Dimension,

tests/append.rs

@@ -18,7 +18,10 @@ fn append_row() {
     assert_eq!(a.try_append_column(aview1(&[1.])),
         Err(ShapeError::from_kind(ErrorKind::IncompatibleShape)));
     assert_eq!(a.try_append_column(aview1(&[1., 2.])),
-        Err(ShapeError::from_kind(ErrorKind::IncompatibleLayout)));
+        Ok(()));
+    assert_eq!(a,
+        array![[0., 1., 2., 3., 1.],
+               [4., 5., 6., 7., 2.]]);
 }
 
 #[test]
@@ -28,8 +31,7 @@ fn append_row_wrong_layout() {
     a.try_append_row(aview1(&[4., 5., 6., 7.])).unwrap();
     assert_eq!(a.shape(), &[2, 4]);
 
-    assert_eq!(a.try_append_column(aview1(&[1., 2.])),
-        Err(ShapeError::from_kind(ErrorKind::IncompatibleLayout)));
+    //assert_eq!(a.try_append_column(aview1(&[1., 2.])), Err(ShapeError::from_kind(ErrorKind::IncompatibleLayout)));
 
     assert_eq!(a,
         array![[0., 1., 2., 3.],
@@ -56,7 +58,13 @@ fn append_row_error() {
     assert_eq!(a.try_append_column(aview1(&[1.])),
         Err(ShapeError::from_kind(ErrorKind::IncompatibleShape)));
     assert_eq!(a.try_append_column(aview1(&[1., 2., 3.])),
-        Err(ShapeError::from_kind(ErrorKind::IncompatibleLayout)));
+        Ok(()));
+    assert_eq!(a.t(),
+        array![[0., 0., 0.],
+               [0., 0., 0.],
+               [0., 0., 0.],
+               [0., 0., 0.],
+               [1., 2., 3.]]);
 }
 
 #[test]
@@ -76,7 +84,11 @@ fn append_row_existing() {
     assert_eq!(a.try_append_column(aview1(&[1.])),
         Err(ShapeError::from_kind(ErrorKind::IncompatibleShape)));
     assert_eq!(a.try_append_column(aview1(&[1., 2., 3.])),
-        Err(ShapeError::from_kind(ErrorKind::IncompatibleLayout)));
+        Ok(()));
+    assert_eq!(a,
+        array![[0., 0., 0., 0., 1.],
+               [0., 1., 2., 3., 2.],
+               [4., 5., 6., 7., 3.]]);
 }
 
 #[test]
@@ -87,8 +99,7 @@ fn append_row_col_len_1() {
     a.try_append_column(aview1(&[2., 3.])).unwrap(); // shape 2 x 2
     assert_eq!(a.try_append_row(aview1(&[1.])),
         Err(ShapeError::from_kind(ErrorKind::IncompatibleShape)));
-    assert_eq!(a.try_append_row(aview1(&[1., 2.])),
-        Err(ShapeError::from_kind(ErrorKind::IncompatibleLayout)));
+    //assert_eq!(a.try_append_row(aview1(&[1., 2.])), Err(ShapeError::from_kind(ErrorKind::IncompatibleLayout)));
     a.try_append_column(aview1(&[4., 5.])).unwrap(); // shape 2 x 3
     assert_eq!(a.shape(), &[2, 3]);
 
@@ -240,3 +251,7 @@ fn test_append_zero_size() {
         assert_eq!(a.shape(), &[0, 2]);
     }
 }
+
+#[test]
+fn move_into() {
+}