Bug#842513: vlc: immediate crash on launch on powerpc

[Lennart Sorensen]

On Fri, Nov 04, 2016 at 01:33:36PM -0400, Lennart Sorensen wrote:
> And of course I made a typo while copying things into the patch.
> 
> Doing another build test of it now.
> 
> --- vlc-2.2.4.orig/configure.ac	2016-05-31 12:11:07.000000000 -0400
> +++ vlc-2.2.4/configure.ac	2016-11-04 12:22:02.543265439 -0400
> @@ -1422,25 +1422,24 @@
>    VLC_SAVE_FLAGS
>    AC_CACHE_CHECK([if \$CC groks AltiVec C extensions],
>    [ac_cv_c_altivec], [
> -    CFLAGS="${CFLAGS} -maltivec"
> +    CFLAGS="${CFLAGS} -maltivec -fno-tree-vectorize"
>      AC_COMPILE_IFELSE([AC_LANG_PROGRAM([
>  [#include <altivec.h>]], [
>  [vec_ld(0, (unsigned char *)0);]])], [
> -      ac_cv_c_altivec="-maltivec"
> +      ac_cv_c_altivec="-maltivec -fno-tree-vectorize"
>      ], [
>        ac_cv_c_altivec="no"
>      ])
>    ])
> -  VLC_RESTORE_FLAGS
>    AS_IF([test "${ac_cv_c_altivec}" != "no"], [
>      CPPFLAGS="${CPPFLAGS} ${ac_cv_c_altivec}"
>      AC_DEFINE(CAN_COMPILE_C_ALTIVEC, 1, [Define to 1 if C AltiVec extensions are available.])
> -    VLC_ADD_CFLAGS([libvlccore],[${ac_cv_c_altivec}])
> -    ALTIVEC_CFLAGS="$ALTIVEC_FLAGS ${ac_cv_c_altivec} ${ac_cv_c_altivec_abi}"
> -    VLC_ADD_CFLAGS([deinterlace],[${ac_cv_c_altivec} ${ac_cv_c_altivec_abi}])
> +    ALTIVEC_CFLAGS="$ALTIVEC_CFLAGS ${ac_cv_c_altivec} ${ac_cv_c_altivec_abi}"
> +    VLC_ADD_CFLAGS([libdeinterlace],[${ac_cv_c_altivec} ${ac_cv_c_altivec_abi}])
>      have_altivec="yes"
>    ])
>    AC_CHECK_HEADERS(altivec.h)
> +  VLC_RESTORE_FLAGS
>  
>    VLC_SAVE_FLAGS
>    LDFLAGS="${LDFLAGS} -Wl,-framework,vecLib"

That did not quite work, so I had to try the slightly more complicated
patch.

This patch works:

--- vlc-2.2.4.orig/configure.ac 2016-05-31 12:11:07.000000000 -0400
+++ vlc-2.2.4/configure.ac      2016-11-08 10:28:54.763640362 -0500
@@ -1422,25 +1422,23 @@
   VLC_SAVE_FLAGS
   AC_CACHE_CHECK([if \$CC groks AltiVec C extensions],
   [ac_cv_c_altivec], [
-    CFLAGS="${CFLAGS} -maltivec"
+    CFLAGS="${CFLAGS} -maltivec -fno-tree-vectorize"
     AC_COMPILE_IFELSE([AC_LANG_PROGRAM([
 [#include <altivec.h>]], [
 [vec_ld(0, (unsigned char *)0);]])], [
-      ac_cv_c_altivec="-maltivec"
+      ac_cv_c_altivec="-maltivec -fno-tree-vectorize"
     ], [
       ac_cv_c_altivec="no"
     ])
   ])
-  VLC_RESTORE_FLAGS
   AS_IF([test "${ac_cv_c_altivec}" != "no"], [
     CPPFLAGS="${CPPFLAGS} ${ac_cv_c_altivec}"
     AC_DEFINE(CAN_COMPILE_C_ALTIVEC, 1, [Define to 1 if C AltiVec extensions are available.])
-    VLC_ADD_CFLAGS([libvlccore],[${ac_cv_c_altivec}])
-    ALTIVEC_CFLAGS="$ALTIVEC_FLAGS ${ac_cv_c_altivec} ${ac_cv_c_altivec_abi}"
-    VLC_ADD_CFLAGS([deinterlace],[${ac_cv_c_altivec} ${ac_cv_c_altivec_abi}])
+    ALTIVEC_CFLAGS="$ALTIVEC_CFLAGS ${ac_cv_c_altivec} ${ac_cv_c_altivec_abi}"
     have_altivec="yes"
   ])
   AC_CHECK_HEADERS(altivec.h)
+  VLC_RESTORE_FLAGS
 
   VLC_SAVE_FLAGS
   LDFLAGS="${LDFLAGS} -Wl,-framework,vecLib"
--- vlc-2.2.4.orig/modules/video_filter/Modules.am      2015-02-02 14:42:29.000000000 -0500
+++ vlc-2.2.4/modules/video_filter/Modules.am   2016-11-08 10:28:31.391639060 -0500
@@ -29,6 +29,7 @@
 libdeinterlace_plugin_la_SOURCES += deinterlace/merge_arm.S
 libdeinterlace_plugin_la_CFLAGS += -DCAN_COMPILE_ARM
 endif
+libdeinterlace_plugin_la_CFLAGS += $(ALTIVEC_CFLAGS)
 video_filter_LTLIBRARIES += libdeinterlace_plugin.la
 
 libdynamicoverlay_plugin_la_SOURCES = \

With this patch, only functions inside a CPU feature check for altivec
support has any vector instructions.

And I realized the reason it was broken is that when using -maltivec and
-O4 (which vlc uses), you get -ftree-vectorize automatically enabled
which means gcc starts to automatically generate vector instructions
all over the place, which is not desired in this case.  It rather defeats
the purpose of having a cpu feature check after all.

-- 
Len Sorensen