Merge pull request #1150 from ychin/apple-silicon-support

ychin · web-flow · commit 5ba08b4af359 · 2020-12-22T21:39:44.000-08:00
Support for building universal x86 / Apple Silicon (arm64) app
diff --git a/.github/workflows/ci-macvim.yaml b/.github/workflows/ci-macvim.yaml
@@ -18,7 +18,10 @@ env:
   vi_cv_dll_name_perl: /System/Library/Perl/5.18/darwin-thread-multi-2level/CORE/libperl.dylib
   vi_cv_dll_name_python: /System/Library/Frameworks/Python.framework/Versions/2.7/Python
   vi_cv_dll_name_python3: /usr/local/Frameworks/Python.framework/Versions/3.9/Python
+  vi_cv_dll_name_python3_arm64: /opt/homebrew/Frameworks/Python.framework/Versions/3.9/Python
   vi_cv_dll_name_ruby: /usr/local/opt/ruby/lib/libruby.dylib
+  vi_cv_dll_name_ruby_arm64: /opt/homebrew/opt/ruby/lib/libruby.dylib
+  vi_cv_dll_name_lua_arm64: /opt/homebrew/lib/liblua.dylib
 
   VIM_BIN: src/MacVim/build/Release/MacVim.app/Contents/MacOS/Vim
   MACVIM_BIN: src/MacVim/build/Release/MacVim.app/Contents/MacOS/MacVim
@@ -38,18 +41,27 @@ jobs:
         - os: macos-10.15
           xcode: 11.7
         - os: macos-10.15
-          publish: true
         - os: macos-11.0
+          publish: true
 
     runs-on: ${{ matrix.os }}
 
     steps:
       - uses: actions/checkout@v2
 
-      # Set up and install gettext for localization.
+      # Set up, install, and cache gettext library for localization.
+      #
       # Instead of using the default binary installed by Homebrew, need to build our own because gettext is statically
       # linked in MacVim, and need to be built against MACOSX_DEPLOYMENT_TARGET to ensure the built binary will work on
       # supported macOS versions.
+      #
+      # In addition, to support building a universal MacVim, we need an arm64 version of gettext as well in order to
+      # create a universal gettext binary to link against (Homebrew only distributes thin binaries and therefore this
+      # has to be done manually). To do that, we will just pull the bottle directly from Homebrew and patch it in using
+      # lipo. We can't use normal brew commands to get the bottle because brew doesn't natively support cross-compiling
+      # and we are running CI on x86_64 Macs. We also don't need to worry about the min deployment target fix on arm64
+      # because all Apple Silicon Macs have to run on macOS 11+.
+
       - name: Set up gettext
         if: matrix.publish
         run: |
@@ -71,11 +83,12 @@ jobs:
           brew uninstall --ignore-dependencies gettext
 
       - name: Cache gettext
+        id: cache-gettext
         if: matrix.publish
         uses: actions/cache@v2
         with:
           path: /usr/local/Cellar/gettext
-          key: gettext-homebrew-cache-${{ runner.os }}-${{ hashFiles('gettext.rb') }}
+          key: gettext-homebrew-cache-patched-unified-${{ hashFiles('gettext.rb') }}
 
       - name: Install gettext
         if: matrix.publish
@@ -85,6 +98,37 @@ jobs:
           brew install -s gettext.rb  # This will be a no-op if gettext was cached
           brew link gettext           # If gettext was cached, this step is necessary to relink it to /usr/local/
 
+      - name: Create universal gettext with arm64 bottle
+        if: matrix.publish && steps.cache-gettext.outputs.cache-hit != 'true'
+        env:
+          HOMEBREW_NO_AUTO_UPDATE: 1
+        run: |
+          set -o verbose
+
+          # Manually download and extract gettext bottle for arm64
+          gettext_url=$(brew info --json gettext | ruby -rjson -e 'j = JSON.parse(STDIN.read); puts j[0]["bottle"]["stable"]["files"]["arm64_big_sur"]["url"]')
+          gettext_ver=$(brew info --json gettext | ruby -rjson -e 'j = JSON.parse(STDIN.read); puts j[0]["versions"]["stable"]')
+
+          mkdir gettext_download
+          cd gettext_download
+          wget --no-verbose ${gettext_url}
+          tar xf gettext*.tar.gz
+
+          # Just for diagnostics, print out the old archs. This should be a thin binary (x86_64)
+          lipo -info /usr/local/lib/libintl.a
+          lipo -info /usr/local/lib/libintl.dylib
+
+          # Create a universal binary by patching the custom built x86_64 one with the downloaded arm64 one.
+          # Modify the actual binaries in /usr/local/Cellar instead of the symlinks to allow caching to work.
+          lipo -create -output /usr/local/Cellar/gettext/${gettext_ver}/lib/libintl.a /usr/local/Cellar/gettext/${gettext_ver}/lib/libintl.a ./gettext/${gettext_ver}/lib/libintl.a
+          lipo -create -output /usr/local/Cellar/gettext/${gettext_ver}/lib/libintl.dylib /usr/local/Cellar/gettext/${gettext_ver}/lib/libintl.dylib ./gettext/${gettext_ver}/lib/libintl.dylib
+
+          # Print out the new archs and verify they are universal with 2 archs.
+          lipo -info /usr/local/lib/libintl.a | grep 'x86_64 arm64'
+          lipo -info /usr/local/lib/libintl.dylib | grep 'x86_64 arm64'
+
+      # Set up remaining packages and tools
+
       - name: Install packages
         if: matrix.publish
         env:
@@ -101,6 +145,8 @@ jobs:
           sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode }}.app/Contents/Developer
           xcode-select -p
 
+      # All set up steps are done. Build and test MacVim below.
+
       - name: Configure
         run: |
           set -o verbose
@@ -111,7 +157,6 @@ jobs:
             --with-tlib=ncurses
             --enable-cscope
             --enable-gui=macvim
-            --with-macarchs=x86_64
             --with-compiledby="GitHub Actions"
           )
           if ${{ matrix.publish == true }}; then
@@ -122,6 +167,11 @@ jobs:
               --enable-rubyinterp=dynamic
               --enable-luainterp=dynamic
               --with-lua-prefix=/usr/local
+              --with-macarchs="x86_64 arm64"
+            )
+          else
+            CONFOPT+=(
+              --with-macarchs=x86_64
             )
           fi
           echo "CONFOPT: ${CONFOPT[@]}"
@@ -140,6 +190,12 @@ jobs:
           grep -q -- "-DDYNAMIC_PYTHON3_DLL=\\\\\"${vi_cv_dll_name_python3}\\\\\"" src/auto/config.mk
           grep -q -- "-DDYNAMIC_RUBY_DLL=\\\\\"${vi_cv_dll_name_ruby}\\\\\"" src/auto/config.mk
 
+          # Also search for the arm64 overrides for the default library locations, which are different from x86_64
+          # because Homebrew puts them at a different place.
+          grep -q -- "-DDYNAMIC_PYTHON3_DLL_ARM64=\\\\\"${vi_cv_dll_name_python3_arm64}\\\\\"" src/auto/config.mk
+          grep -q -- "-DDYNAMIC_RUBY_DLL_ARM64=\\\\\"${vi_cv_dll_name_ruby_arm64}\\\\\"" src/auto/config.mk
+          grep -q -- "-DDYNAMIC_LUA_DLL_ARM64=\\\\\"${vi_cv_dll_name_lua_arm64}\\\\\"" src/auto/config.mk
+
       - name: Show configure output
         run: |
           cat src/auto/config.mk
@@ -185,12 +241,11 @@ jobs:
             echo 'Found external dynamic linkage!'; false
           fi
 
-          # Make sure we are building x86_64 only. arm64 builds don't work properly now, so we don't want to accidentally build
-          # it as it will get prioritized by Apple Silicon Macs.
+          # Make sure we are building universal x86_64 / arm64 builds and didn't accidentally create a thin app.
           check_arch() {
             local archs=($(lipo -archs "$1"))
-            if [[ ${archs[@]} != x86_64 ]]; then
-              echo "Found unexpected arch(s) in $1: ${archs[@]}"; false
+            if [[ ${archs[@]} != "x86_64 arm64" ]]; then
+              echo "Wrong arch(s) in $1: ${archs[@]}"; false
             fi
           }
           check_arch "${VIM_BIN}"
diff --git a/src/auto/configure b/src/auto/configure
@@ -5673,6 +5673,20 @@ $as_echo "yes" >&6; }
 
       LUA_LIBS=""
       LUA_CFLAGS="-DDYNAMIC_LUA_DLL=\\\"${vi_cv_dll_name_lua}\\\" $LUA_CFLAGS"
+
+      # MacVim patch to hack in a different default dynamic lib path for
+      # arm64. We don't test that it links here so this has to be binary
+      # compatible with DYNAMIC_LUA_DLL
+      { $as_echo "$as_me:${as_lineno-$LINENO}: checking liblua${luajit}*.${ext}* (arm64)" >&5
+$as_echo_n "checking liblua${luajit}*.${ext}* (arm64)... " >&6; }
+      if test -n "${vi_cv_dll_name_lua_arm64}"; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: ${vi_cv_dll_name_lua_arm64}" >&5
+$as_echo "${vi_cv_dll_name_lua_arm64}" >&6; }
+        LUA_CFLAGS+=" -DDYNAMIC_LUA_DLL_ARM64=\\\"${vi_cv_dll_name_lua_arm64}\\\""
+      else
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: <none>" >&5
+$as_echo "<none>" >&6; }
+      fi
     fi
     if test "X$LUA_CFLAGS$LUA_LIBS" != "X" && \
        test "x$MACOS_X" = "xyes" && test "x$vi_cv_with_luajit" != "xno" && \
@@ -7178,6 +7192,20 @@ fi
   PYTHON3_OBJ="objects/if_python3.o"
   PYTHON3_CFLAGS="$PYTHON3_CFLAGS -DDYNAMIC_PYTHON3_DLL=\\\"${vi_cv_dll_name_python3}\\\""
   PYTHON3_LIBS=
+
+  # MacVim patch to hack in a different default dynamic lib path for arm64.
+  # We don't test that it links here so this has to be binary compatible with
+  # DYNAMIC_PYTHON3_DLL
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking Python3's dll name (arm64)" >&5
+$as_echo_n "checking Python3's dll name (arm64)... " >&6; }
+  if test -n "${vi_cv_dll_name_python3_arm64}"; then
+    { $as_echo "$as_me:${as_lineno-$LINENO}: result: ${vi_cv_dll_name_python3_arm64}" >&5
+$as_echo "${vi_cv_dll_name_python3_arm64}" >&6; }
+    PYTHON3_CFLAGS+=" -DDYNAMIC_PYTHON3_DLL_ARM64=\\\"${vi_cv_dll_name_python3_arm64}\\\""
+  else
+    { $as_echo "$as_me:${as_lineno-$LINENO}: result: <none>" >&5
+$as_echo "<none>" >&6; }
+  fi
 elif test "$python_ok" = yes && test "$enable_pythoninterp" = "dynamic"; then
   $as_echo "#define DYNAMIC_PYTHON 1" >>confdefs.h
 
@@ -7224,6 +7252,20 @@ elif test "$python3_ok" = yes && test "$enable_python3interp" = "dynamic"; then
   PYTHON3_OBJ="objects/if_python3.o"
   PYTHON3_CFLAGS="$PYTHON3_CFLAGS -DDYNAMIC_PYTHON3_DLL=\\\"${vi_cv_dll_name_python3}\\\""
   PYTHON3_LIBS=
+
+  # MacVim patch to hack in a different default dynamic lib path for arm64.
+  # We don't test that it links here so this has to be binary compatible with
+  # DYNAMIC_PYTHON3_DLL
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking Python3's dll name (arm64)" >&5
+$as_echo_n "checking Python3's dll name (arm64)... " >&6; }
+  if test -n "${vi_cv_dll_name_python3_arm64}"; then
+    { $as_echo "$as_me:${as_lineno-$LINENO}: result: ${vi_cv_dll_name_python3_arm64}" >&5
+$as_echo "${vi_cv_dll_name_python3_arm64}" >&6; }
+    PYTHON3_CFLAGS+=" -DDYNAMIC_PYTHON3_DLL_ARM64=\\\"${vi_cv_dll_name_python3_arm64}\\\""
+  else
+    { $as_echo "$as_me:${as_lineno-$LINENO}: result: <none>" >&5
+$as_echo "<none>" >&6; }
+  fi
 elif test "$python3_ok" = yes; then
       { $as_echo "$as_me:${as_lineno-$LINENO}: checking if -fPIE can be added for Python3" >&5
 $as_echo_n "checking if -fPIE can be added for Python3... " >&6; }
@@ -7763,6 +7805,23 @@ $as_echo "$rubyhdrdir" >&6; }
 
 	  RUBY_CFLAGS="-DDYNAMIC_RUBY_DLL=\\\"$libruby_soname\\\" $RUBY_CFLAGS"
 	  RUBY_LIBS=
+
+      # MacVim patch to hack in a different default dynamic lib path for
+      # arm64.  We don't test that it links here so this has to be binary
+      # compatible with DYNAMIC_RUBY_DLL
+      # Note: Apple does ship with a default Ruby lib, but it's usually older
+      # than Homebrew, and since on x86_64 we use the Homebrew version, we
+      # should use that as well for Apple Silicon.
+      { $as_echo "$as_me:${as_lineno-$LINENO}: checking ${libruby_soname} (arm64)" >&5
+$as_echo_n "checking ${libruby_soname} (arm64)... " >&6; }
+      if test -n "${vi_cv_dll_name_ruby_arm64}"; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: ${vi_cv_dll_name_ruby_arm64}" >&5
+$as_echo "${vi_cv_dll_name_ruby_arm64}" >&6; }
+        RUBY_CFLAGS+=" -DDYNAMIC_RUBY_DLL_ARM64=\\\"${vi_cv_dll_name_ruby_arm64}\\\""
+      else
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: <none>" >&5
+$as_echo "<none>" >&6; }
+      fi
 	fi
       else
 	{ $as_echo "$as_me:${as_lineno-$LINENO}: result: not found; disabling Ruby" >&5
diff --git a/src/configure.ac b/src/configure.ac
@@ -777,6 +777,17 @@ if test "$enable_luainterp" = "yes" -o "$enable_luainterp" = "dynamic"; then
       AC_DEFINE(DYNAMIC_LUA)
       LUA_LIBS=""
       LUA_CFLAGS="-DDYNAMIC_LUA_DLL=\\\"${vi_cv_dll_name_lua}\\\" $LUA_CFLAGS"
+
+      # MacVim patch to hack in a different default dynamic lib path for
+      # arm64. We don't test that it links here so this has to be binary
+      # compatible with DYNAMIC_LUA_DLL
+      AC_MSG_CHECKING([liblua${luajit}*.${ext}* (arm64)])
+      if test -n "${vi_cv_dll_name_lua_arm64}"; then
+        AC_MSG_RESULT([${vi_cv_dll_name_lua_arm64}])
+        LUA_CFLAGS+=" -DDYNAMIC_LUA_DLL_ARM64=\\\"${vi_cv_dll_name_lua_arm64}\\\""
+      else
+        AC_MSG_RESULT([<none>])
+      fi
     fi
     if test "X$LUA_CFLAGS$LUA_LIBS" != "X" && \
        test "x$MACOS_X" = "xyes" && test "x$vi_cv_with_luajit" != "xno" && \
@@ -1799,6 +1810,17 @@ if test "$python_ok" = yes && test "$python3_ok" = yes; then
   PYTHON3_OBJ="objects/if_python3.o"
   PYTHON3_CFLAGS="$PYTHON3_CFLAGS -DDYNAMIC_PYTHON3_DLL=\\\"${vi_cv_dll_name_python3}\\\""
   PYTHON3_LIBS=
+
+  # MacVim patch to hack in a different default dynamic lib path for arm64.
+  # We don't test that it links here so this has to be binary compatible with
+  # DYNAMIC_PYTHON3_DLL
+  AC_MSG_CHECKING([Python3's dll name (arm64)])
+  if test -n "${vi_cv_dll_name_python3_arm64}"; then
+    AC_MSG_RESULT([${vi_cv_dll_name_python3_arm64}])
+    PYTHON3_CFLAGS+=" -DDYNAMIC_PYTHON3_DLL_ARM64=\\\"${vi_cv_dll_name_python3_arm64}\\\""
+  else
+    AC_MSG_RESULT([<none>])
+  fi
 elif test "$python_ok" = yes && test "$enable_pythoninterp" = "dynamic"; then
   AC_DEFINE(DYNAMIC_PYTHON)
   PYTHON_SRC="if_python.c"
@@ -1827,6 +1849,17 @@ elif test "$python3_ok" = yes && test "$enable_python3interp" = "dynamic"; then
   PYTHON3_OBJ="objects/if_python3.o"
   PYTHON3_CFLAGS="$PYTHON3_CFLAGS -DDYNAMIC_PYTHON3_DLL=\\\"${vi_cv_dll_name_python3}\\\""
   PYTHON3_LIBS=
+
+  # MacVim patch to hack in a different default dynamic lib path for arm64.
+  # We don't test that it links here so this has to be binary compatible with
+  # DYNAMIC_PYTHON3_DLL
+  AC_MSG_CHECKING([Python3's dll name (arm64)])
+  if test -n "${vi_cv_dll_name_python3_arm64}"; then
+    AC_MSG_RESULT([${vi_cv_dll_name_python3_arm64}])
+    PYTHON3_CFLAGS+=" -DDYNAMIC_PYTHON3_DLL_ARM64=\\\"${vi_cv_dll_name_python3_arm64}\\\""
+  else
+    AC_MSG_RESULT([<none>])
+  fi
 elif test "$python3_ok" = yes; then
   dnl Check that adding -fPIE works.  It may be needed when using a static
   dnl Python library.
@@ -2084,6 +2117,20 @@ if test "$enable_rubyinterp" = "yes" -o "$enable_rubyinterp" = "dynamic"; then
 	  AC_DEFINE(DYNAMIC_RUBY)
 	  RUBY_CFLAGS="-DDYNAMIC_RUBY_DLL=\\\"$libruby_soname\\\" $RUBY_CFLAGS"
 	  RUBY_LIBS=
+
+      # MacVim patch to hack in a different default dynamic lib path for
+      # arm64.  We don't test that it links here so this has to be binary
+      # compatible with DYNAMIC_RUBY_DLL
+      # Note: Apple does ship with a default Ruby lib, but it's usually older
+      # than Homebrew, and since on x86_64 we use the Homebrew version, we
+      # should use that as well for Apple Silicon.
+      AC_MSG_CHECKING([${libruby_soname} (arm64)])
+      if test -n "${vi_cv_dll_name_ruby_arm64}"; then
+        AC_MSG_RESULT([${vi_cv_dll_name_ruby_arm64}])
+        RUBY_CFLAGS+=" -DDYNAMIC_RUBY_DLL_ARM64=\\\"${vi_cv_dll_name_ruby_arm64}\\\""
+      else
+        AC_MSG_RESULT([<none>])
+      fi
 	fi
       else
 	AC_MSG_RESULT(not found; disabling Ruby)
diff --git a/src/optiondefs.h b/src/optiondefs.h
@@ -309,6 +309,36 @@ struct vimoption
 # define DEFAULT_PYTHON_VER	0
 #endif
 
+// Support targeting different dynamic linkages for scripting languages based on
+// arch on macOS. This is necessary because package managers such as Homebrew
+// distributes thin binaries, and therefore the x86_64 and arm64 libraries are
+// located in different places.
+#ifdef MACOS_X
+# if defined(DYNAMIC_PYTHON3_DLL_X86_64) && defined(__x86_64__)
+#  undef DYNAMIC_PYTHON3_DLL
+#  define DYNAMIC_PYTHON3_DLL DYNAMIC_PYTHON3_DLL_X86_64
+# elif defined(DYNAMIC_PYTHON3_DLL_ARM64) && defined(__arm64__)
+#  undef DYNAMIC_PYTHON3_DLL
+#  define DYNAMIC_PYTHON3_DLL DYNAMIC_PYTHON3_DLL_ARM64
+# endif
+
+# if defined(DYNAMIC_RUBY_DLL_X86_64) && defined(__x86_64__)
+#  undef DYNAMIC_RUBY_DLL
+#  define DYNAMIC_RUBY_DLL DYNAMIC_RUBY_DLL_X86_64
+# elif defined(DYNAMIC_RUBY_DLL_ARM64) && defined(__arm64__)
+#  undef DYNAMIC_RUBY_DLL
+#  define DYNAMIC_RUBY_DLL DYNAMIC_RUBY_DLL_ARM64
+# endif
+
+# if defined(DYNAMIC_LUA_DLL_X86_64) && defined(__x86_64__)
+#  undef DYNAMIC_LUA_DLL
+#  define DYNAMIC_LUA_DLL DYNAMIC_LUA_DLL_X86_64
+# elif defined(DYNAMIC_LUA_DLL_ARM64) && defined(__arm64__)
+#  undef DYNAMIC_LUA_DLL
+#  define DYNAMIC_LUA_DLL DYNAMIC_LUA_DLL_ARM64
+# endif
+#endif
+
 // used for 'cinkeys' and 'indentkeys'
 #define INDENTKEYS_DEFAULT (char_u *)"0{,0},0),0],:,0#,!^F,o,O,e"
 
