Segfault by AVX misalignment with Vector4d/Quaternion type

Submitted by Mark Sauder

Assigned to Nobody

Link to original bugzilla bug (#1233)
Version: 3.3 (current stable)
Operating system: Linux

Description

When manipulating Quaterniond typedefs, an alignment error is occurring
which causes a perfectly repeatable segfault during each program run.

Attempts at re-typecasting to Vector4d types higher upstream and/or re-assignment of the upstream variables element by element manually to new quaternion variables changes the point at which the segfault occurs.

Commit b1bd53aa introduced this behavior, b1bd53aa.

Here is a gdb backtrace of the event:

Thread 2 received signal SIGSEGV, Segmentation fault.

[Switching to Thread 0x7ffff6f4f700 (LWP 21643)]  
0x0000000000428045 in _mm256_store_pd (__A=..., __P=<optimized out>)  
    at /usr/lib/gcc/x86_64-linux-gnu/5/include/avxintrin.h:842  
842	  *(__m256d *)__P = __A;  
(gdb) backtrace  
#0  0x0000000000428045 in _mm256_store_pd (__A=..., __P=<optimized out>)  
    at /usr/lib/gcc/x86_64-linux-gnu/5/include/avxintrin.h:842  
#1  Eigen::internal::pstore<double, double __vector(4)>(double*, double __vector(4) const&) (from=..., to=<optimized out>)  
    at /usr/include/eigen3/Eigen/src/Core/arch/AVX/PacketMath.h:245  
#2  Eigen::internal::pstoret<double, double __vector(4), 32>(double*, double __vector(4) const&) (from=..., to=<optimized out>)  
    at /usr/include/eigen3/Eigen/src/Core/GenericPacketMath.h:516  
#3  Eigen::internal::assign_op<double>::assignPacket<32, double __vector(4)>(double*, double __vector(4) const&) const (b=..., a=<optimized out>,   
    this=<optimized out>)  
    at /usr/include/eigen3/Eigen/src/Core/functors/AssignmentFunctors.h:28  
#4  Eigen::internal::generic_dense_assignment_kernel<Eigen::internal::evaluator<Eigen::Matrix<double, 4, 1, 0, 4, 1> >, Eigen::internal::evaluator<Eigen::Matrix<double, 4, 1, 0, 4, 1> >, Eigen::internal::assign_op<double>, 0>::assignPacket<32, 32, double __vector(4)>(long, long) (col=<optimized out>,   
    row=<optimized out>, this=<synthetic pointer>)  
    at /usr/include/eigen3/Eigen/src/Core/AssignEvaluator.h:619  
#5  Eigen::internal::generic_dense_assignment_kernel<Eigen::internal::evaluator<Eigen::Matrix<double, 4, 1, 0, 4, 1> >, Eigen::internal::evaluator<Eigen::Matrix<double, 4, 1, 0, 4, 1> >, Eigen::internal::assign_op<double>, 0>::assignPacketByOuterInner<32, 32, double __vector(4)>(long, long) (inner=0, outer=0,   
    this=<synthetic pointer>)  
    at /usr/include/eigen3/Eigen/src/Core/AssignEvaluator.h:633  
#6  Eigen::internal::copy_using_evaluator_innervec_CompleteUnrolling<Eigen::internal::generic_dense_assignment_kernel<Eigen::internal::evaluator<Eigen::Matrix<double, 4, 1, 0, 4, 1> >, Eigen::internal::evaluator<Eigen::Matrix<double, 4, 1, 0, 4, 1> >, Eigen::internal::assign_op<double>, 0>, 0, 4>::run (  
    kernel=<synthetic pointer>, kernel=<synthetic pointer>)  
    at /usr/include/eigen3/Eigen/src/Core/AssignEvaluator.h:265  
#7  Eigen::internal::dense_assignment_loop<Eigen::internal::generic_dense_assignment_kernel<Eigen::internal::evaluator<Eigen::Matrix<double, 4, 1, 0, 4, 1> >, Eigen::internal::evaluator<Eigen::Matrix<double, 4, 1, 0, 4, 1> >, Eigen::internal::assign_op<double>, 0>, 2, 2>::run (kernel=<synthetic pointer>)  
    at /usr/include/eigen3/Eigen/src/Core/AssignEvaluator.h:460  
#8  Eigen::internal::call_dense_assignment_loop<Eigen::Matrix<double, 4, 1, 0, 4, 1>, Eigen::Matrix<double, 4, 1, 0, 4, 1>, Eigen::internal::assign_op<double> > (func=..., src=..., dst=...)  
    at /usr/include/eigen3/Eigen/src/Core/AssignEvaluator.h:680  
#9  Eigen::internal::Assignment<Eigen::Matrix<double, 4, 1, 0, 4, 1>, Eigen::Matrix<double, 4, 1, 0, 4, 1>, Eigen::internal::assign_op<double>, Eigen::internal::Dense2Dense, double>::run (func=..., src=..., dst=...)  
    at /usr/include/eigen3/Eigen/src/Core/AssignEvaluator.h:827  
#10 Eigen::internal::call_assignment_no_alias<Eigen::Matrix<double, 4, 1, 0, 4, 1>, Eigen::Matrix<double, 4, 1, 0, 4, 1>, Eigen::internal::assign_op<double> >  
    (func=..., src=..., dst=...)  
    at /usr/include/eigen3/Eigen/src/Core/AssignEvaluator.h:780  
#11 Eigen::internal::call_assignment<Eigen::Matrix<double, 4, 1, 0, 4, 1>, Eigen::Matrix<double, 4, 1, 0, 4, 1>, Eigen::internal::assign_op<double> >(Eigen::Matrix<double, 4, 1, 0, 4, 1>&, Eigen::Matrix<double, 4, 1, 0, 4, 1> const&, Eigen::internal::assign_op<double> const&, Eigen::internal::enable_if<!Eigen::internal::evaluator_assume_aliasing<Eigen::Matrix<double, 4, 1, 0, 4, 1>, Eigen::internal::evaluator_traits<Eigen::Matrix<double, 4, 1, 0, 4, 1> >::Shape>::value, void*>::type) (func=..., src=..., dst=...)  
    at /usr/include/eigen3/Eigen/src/Core/AssignEvaluator.h:743  
#12 Eigen::internal::call_assignment<Eigen::Matrix<double, 4, 1, 0, 4, 1>, Eigen::Matrix<double, 4, 1, 0, 4, 1> > (src=..., dst=...)  
    at /usr/include/eigen3/Eigen/src/Core/AssignEvaluator.h:721  
#13 Eigen::PlainObjectBase<Eigen::Matrix<double, 4, 1, 0, 4, 1> >::_set<Eigen::Matrix<double, 4, 1, 0, 4, 1> > (other=..., this=0x7ffff003b590)  
    at /usr/include/eigen3/Eigen/src/Core/PlainObjectBase.h:682  
#14 Eigen::Matrix<double, 4, 1, 0, 4, 1>::operator= (other=...,   
    this=0x7ffff003b590) at /usr/include/eigen3/Eigen/src/Core/Matrix.h:208  
#15 Eigen::QuaternionBase<Eigen::Quaternion<double, 0> >::operator= (  
    other=..., this=0x7ffff003b590)  
    at /usr/include/eigen3/Eigen/src/Geometry/Quaternion.h:486  
#16 Eigen::Quaternion<double, 0>::operator= (other=..., this=0x7ffff003b590)  
    at /usr/include/eigen3/Eigen/src/Geometry/Quaternion.h:235

Edited Dec 05, 2019 by Eigen Bugzilla