Merge pull request #1544 from Kazzz-S/0.28.13-mac1

Overhauled the build system for MacOS ['Monterey', 'Ventura', 'Sonoma'] to build KLayout-0.28.13 or later with support for libgit2

macbuild/Resources/start-console.py

@@ -6,6 +6,25 @@
 pwd = os.getcwd()
 sys.path.append(pwd)
 
+piphelpstr = """
+--------------------------------------------------------------------------------
+(1) Install ['pandas', 'scipy', 'matplotlib']
+>>> import pip
+>>> pip.main( ['install', 'pandas', 'scipy', 'matplotlib'] )
+
+(2) List modules
+>>> import pip
+>>> pip.main( ['list'] )
+
+(3) Uninstall ['scipy']
+>>> import pip
+>>> pip.main( ['uninstall', 'scipy'] )
+--------------------------------------------------------------------------------
+"""
+def howtopip():
+    print(piphelpstr)
+
+
 variables = globals().copy()
 variables.update(locals())
 shell = code.InteractiveConsole(variables)

macbuild/Resources/template-start-console.py

@@ -0,0 +1,39 @@
+#!/Applications/klayout.app/Contents/MacOS/klayout -b -r
+import readline
+import code
+import sys
+import os
+pwd = os.getcwd()
+sys.path.append(pwd)
+
+rootPython = "/Applications/klayout.app/Contents/Frameworks/Python.framework/Versions"
+verPython  = "${PYTHON_VER}"
+piptarget  = [ "--target", "%s/%s/lib/python%s/site-packages" % (rootPython, verPython, verPython) ]
+
+piphelpstr = """
+--------------------------------------------------------------------------------
+(1) Install ['pandas', 'scipy', 'matplotlib']
+>>> import pip
+>>> pip.main( ['install', 'pandas', 'scipy', 'matplotlib'] + piptarget )
+
+(2) List modules
+>>> import pip
+>>> pip.main( ['list'] )
+
+(3) Uninstall ['scipy']
+>>> import pip
+>>> pip.main( ['uninstall', 'scipy'] )
+--------------------------------------------------------------------------------
+"""
+def howtopip():
+    print(piphelpstr)
+
+
+variables = globals().copy()
+variables.update(locals())
+shell = code.InteractiveConsole(variables)
+cprt = 'Type "help", "copyright", "credits" or "license" for more information.'
+banner = "Python %s on %s\n%s\n(%s)" % (sys.version, sys.platform,
+    cprt, "KLayout Python Console")
+exit_msg = 'now exiting %s...' % "KLayout Python Console"
+shell.interact(banner, exit_msg)

macbuild/build4mac_util.py

@@ -623,18 +623,24 @@ def Patch_Python_In_PythonFramework( pythonFrameworkPath,
 #----------------------------------------------------------------------------------------
 ## To change the Python's relative library paths to the absolute paths
 #
-# 1: absolute path as in ~python@3.9.17
+# 1: absolute path as seen in ~python@3.9.17
 #      BigSur{kazzz-s} lib-dynload (1)% otool -L _sqlite3.cpython-39-darwin.so
 #      _sqlite3.cpython-39-darwin.so:
 #   ===> /usr/local/opt/sqlite/lib/libsqlite3.0.dylib (compatibility version 9.0.0, current version 9.6.0)
 #        /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1292.100.5)
 #
-# 2: relative path as in  python@3.9.18~
+# 2: relative path as seen in python@3.9.18
 #      Monterey{kazzz-s} lib-dynload (1)% otool -L _sqlite3.cpython-39-darwin.so
 #      _sqlite3.cpython-39-darwin.so:
 #   ===> @loader_path/../../../../../../../../../../opt/sqlite/lib/libsqlite3.0.dylib (compatibility version 9.0.0, current version 9.6.0)
 #        /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1311.100.3)
 #
+# 3. absolute path again as seen in python@3.11
+#      Monterey{kazzz-s} lib-dynload (1)% otool -L _sqlite3.cpython-311-darwin.so
+#      _sqlite3.cpython-311-darwin.so:
+#   ===> /usr/local/opt/sqlite/lib/libsqlite3.0.dylib (compatibility version 9.0.0, current version 9.6.0)
+#        /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1311.100.3)
+#
 # @param[in] frameworkPath: Python Framework path
 # @param[in] debug_level:   debug level
 #
@@ -650,17 +656,25 @@ def Change_Python_LibPath_RelativeToAbsolute( frameworkPath, debug_level=0 ):
     patRel3         = r'(%s)(.+)' % HomebrewSearchPathFilter3   # =    '@loader_path/../../../../../../../../../../opt'
     regRel3         = re.compile(patRel3)
 
-    #----------------------------------------------------------------------
+    #---------------------------------------------------------------------------------------------------
     # (A) Collect *.[so|dylib] that the Python Frameworks depends on
-    #----------------------------------------------------------------------
-    # Ref. https://formulae.brew.sh/formula/python@3.9#default
+    #---------------------------------------------------------------------------------------------------
+    # Ref. https://formulae.brew.sh/formula/python@3.9
     #   as of 2023-09-22, python@3.9 depends on:
     #     gdbm        1.23    GNU database manager
     #     mpdecimal   2.5.1   Library for decimal floating point arithmetic
     #     openssl@3   3.1.2   Cryptography and SSL/TLS Toolkit
     #     readline    8.2.1   Library for command-line editing
     #     sqlite      3.43.1  Command-line interface for SQLite
     #     xz          5.4.4   General-purpose data compression with high compression ratio
+    #---------------------------------------------------------------------------------------------------
+    # https://formulae.brew.sh/formula/python@3.11
+    #   as of 2023-10-24, python@3.11 depends on:
+    #     mpdecimal   2.5.1   Library for decimal floating point arithmetic
+    #     openssl@3   3.1.3   Cryptography and SSL/TLS Toolkit
+    #     sqlite      3.43.2  Command-line interface for SQLite
+    #     xz          5.4.4   General-purpose data compression with high compression ratio
+    #---------------------------------------------------------------------------------------------------
     find_grep_results = os.popen( 'find %s -type f | grep -E "%s"' % (frameworkPath, filter_regex) ).read().split('\n')
     framework_files   = filter( lambda x: x != '', map(lambda x: x.strip(), find_grep_results) )
 
@@ -745,6 +759,37 @@ def Change_Python_LibPath_RelativeToAbsolute( frameworkPath, debug_level=0 ):
     print( "           ---> Change_Python_LibPath_RelativeToAbsolute(): Changed the library paths." )
     return 0
 
+#----------------------------------------------------------------------------------------
+## To generate the 'start-console.py' file from the template file
+#
+# @param[in] template:  input template file (template-start-console.py)
+# @param[in] pythonver: Python version string such as "3.11"
+# @param[in] target:    output target file  (start-console.py)
+#
+# @return True on success, False on failure
+#----------------------------------------------------------------------------------------
+def Generate_Start_Console_Py( template, pythonver, target ):
+    try:
+        fd   = open( template, "r" )
+        tmpl = fd.read()
+        fd.close()
+    except Exception as e:
+        print( "! Failed to read <%s>" % template, file=sys.stderr )
+        return False
+    else:
+        t = string.Template(tmpl)
+        startpy = t.safe_substitute( PYTHON_VER=pythonver )
+
+    try:
+        fd = open( target, "w" )
+        fd.write(startpy)
+        fd.close()
+    except Exception as e:
+        print( "! Failed to write <%s>" % target, file=sys.stderr )
+        return False
+    else:
+        return True
+
 #----------------
 # End of File
 #----------------

macbuild/macQAT.py

@@ -34,6 +34,7 @@ def SetGlobals():
     global WorkDir            # work directory name
     global LogFile            # log file name
     global Usage              # string on usage
+    global DryRun             # dry-run mode; print the command line and exit
     # auxiliary variables on platform
     global System             # 6-tuple from platform.uname()
     global Node               # - do -
@@ -48,28 +49,32 @@ def SetGlobals():
     Usage += "<< Usage of 'macQAT.py' >>\n"
     Usage += "       for running 'ut_runner' after building KLayout.\n"
     Usage += "\n"
-    Usage += "$ [python] ./macQAT.py \n"
+    Usage += "$ [python] ./macQAT.py\n"
     Usage += "   option & argument      : descriptions                            | default value\n"
     Usage += "   -----------------------------------------------------------------+---------------\n"
-    Usage += "   [-u|--usage]           : print usage of 'ut_runner'and exit      | disabled \n"
-    Usage += "                                                                    | \n"
-    Usage += "   <-r|--run>             : run this script                         | disabled \n"
-    Usage += "   [-s|--stop]            : stop on error                           | disabled \n"
-    Usage += "   [-x|--exclude <tests>] : exclude test(s) such as 'pymod,pya'     | '' \n"
-    Usage += "   [-a|--args <string>]   : arguments other than '-x' and '-c'      | '' \n"
-    Usage += "   [-?|--?]               : print this usage and exit               | disabled \n"
+    Usage += "   [-u|--usage]           : print usage of 'ut_runner'and exit      | disabled\n"
+    Usage += "                                                                    |\n"
+    Usage += "   <-r|--run>             : run this script                         | disabled\n"
+    Usage += "   [-s|--stop]            : stop on error                           | disabled\n"
+    Usage += "   [-x|--exclude <tests>] : exclude test(s) such as 'pymod,pya'     | ''\n"
+    Usage += "                          : you can use this option multiple times  |\n"
+    Usage += "   [-a|--args <string>]   : arguments other than '-x' and '-c'      | ''\n"
+    Usage += "                          : you can use this option multiple times  |\n"
+    Usage += "   [--dryrun]             : print the command line and exit         | disabled\n"
+    Usage += "   [-?|--?]               : print this usage and exit               | disabled\n"
     Usage += "--------------------------------------------------------------------+-------------------\n"
 
     ProjectDir      = os.getcwd()
     RunnerUsage     = False
     Run             = False
     ContinueOnError = True
     TestsExcluded   = list()
-    Arguments       = ""
+    Arguments       = list()
     GitSHA1         = GetGitShortSHA1()
     TimeStamp       = GetTimeStamp()
     WorkDir         = "QATest_%s_%s__%s" % (GitSHA1, TimeStamp, os.path.basename(ProjectDir) )
     LogFile         = WorkDir + ".log"
+    DryRun          = False
 
     (System, Node, Release, Version, Machine, Processor) = platform.uname()
 
@@ -103,6 +108,7 @@ def ParseCommandLineArguments():
     global ContinueOnError
     global TestsExcluded
     global Arguments
+    global DryRun
 
     p = optparse.OptionParser( usage=Usage )
     p.add_option( '-u', '--usage',
@@ -124,13 +130,21 @@ def ParseCommandLineArguments():
                     help='stop on error (false)' )
 
     p.add_option( '-x', '--exclude',
+                    action='append',
                     dest='exclude_tests',
                     help="exclude test(s) such as 'pymod,pya' ('')" )
 
     p.add_option( '-a', '--args',
+                    action='append',
                     dest='arguments',
                     help="arguments other than '-x' and '-c' ('')" )
 
+    p.add_option( '--dryrun',
+                    action='store_true',
+                    dest='dryrun',
+                    default=False,
+                    help='print the command line and exit (false)' )
+
     p.add_option( '-?', '--??',
                     action='store_true',
                     dest='checkusage',
@@ -140,8 +154,9 @@ def ParseCommandLineArguments():
     p.set_defaults( runner_usage  = False,
                     runme         = False,
                     stop_on_error = False,
-                    exclude_tests = "",
-                    arguments     = "",
+                    exclude_tests = list(),
+                    arguments     = list(),
+                    dryrun        = False,
                     checkusage    = False )
 
     opt, args = p.parse_args()
@@ -152,11 +167,15 @@ def ParseCommandLineArguments():
     RunnerUsage     = opt.runner_usage
     Run             = opt.runme
     ContinueOnError = not opt.stop_on_error
-    if not opt.exclude_tests == "":
-        TestsExcluded = [ item.strip() for item in opt.exclude_tests.split(',') ]
+    if not len(opt.exclude_tests) == 0:
+        excluded_tests = list()
+        for item in opt.exclude_tests:
+            excluded_tests += [ subitem.strip() for subitem in item.split(',') ]
+        TestsExcluded = sorted(list(set(excluded_tests)))
     else:
         TestsExcluded = []
     Arguments = opt.arguments
+    DryRun    = opt.dryrun
 
 #-------------------------------------------------------------------------------
 ## Hide/Show the private directory
@@ -248,16 +267,20 @@ def Main():
         command += " -c"
     for item in TestsExcluded:
         command += ' -x %s' % item
-    if not Arguments == "":
-        command += " %s" % Arguments
+    if not len(Arguments) == 0:
+        for arg in Arguments:
+            command += " %s" % arg
 
     print( "" )
     print( "### Dumping the log to <%s>" % LogFile )
-    print( "------------------------------------------------------------" )
-    print( "  Git SHA1   = %s" % GitSHA1 )
-    print( "  Time stamp = %s" % TimeStamp )
-    print( "------------------------------------------------------------" )
-    sleep( 1.0 )
+    print( "------------------------------------------------------------------------" )
+    print( "  Git SHA1     = %s" % GitSHA1 )
+    print( "  Time stamp   = %s" % TimeStamp )
+    print( "  Command line = %s" % command )
+    print( "------------------------------------------------------------------------" )
+    if DryRun:
+        quit()
+    sleep(1.0)
     HidePrivateDir()
     RunTester( command, logfile=LogFile )
     ShowPrivateDir()

macbuild/macQAT.sh

@@ -5,7 +5,7 @@
 #
 #  The top Bash script to run "ut_runner" after building KLayout
 #  (http://www.klayout.de/index.php) version 0.26.1 or later on different Apple
-#  ßMac OSX platforms.
+#  Mac OSX platforms.
 #
 #  This script must be copied to a "*.macQAT/" directory to run.
 #===============================================================================
@@ -68,8 +68,8 @@ if [ $# -eq 1 ]; then
         echo " Time stamp = ${timestamp}"
         echo " Log file   = ${logfile}"
         echo " Usage:"
-        echo "  ./QATest.sh -h: to get the help of 'ut_runner'"
-        echo "  ./QATest.sh -r: to run the tests with '-c' option: continues after an error"
+        echo "  ./macQAT.sh -h: to get the help of 'ut_runner'"
+        echo "  ./macQAT.sh -r: to run the tests with '-c' option: continues after an error"
         echo ""
         exit 0
     fi
@@ -79,8 +79,8 @@ else
     echo " Time stamp = ${timestamp}"
     echo " Log file   = ${logfile}"
     echo " Usage:"
-    echo "  ./QATest.sh -h: to get the help of 'ut_runner'"
-    echo "  ./QATest.sh -r: to run the tests with '-c' option: continues after an error"
+    echo "  ./macQAT.sh -h: to get the help of 'ut_runner'"
+    echo "  ./macQAT.sh -r: to run the tests with '-c' option: continues after an error"
     echo ""
     exit 0
 fi

macbuild/macQAT2.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+#===============================================================================
+# File: "macbuild/macQAT2.sh"
+#
+#  The top Bash script to run "ut_runner" after building KLayout
+#  (http://www.klayout.de/index.php) version 0.26.1 or later on different Apple
+#  Mac OSX platforms.
+#
+#  This script must be copied to a directory that can be found in $PATH.
+#===============================================================================
+
+prjdir=`pwd`
+
+if [ $# -ge 2 ]; then
+	first_arg=$1
+	shift
+	remaining_args=("$@")
+	echo "#########################"
+	echo "#  Starging macQAT2.sh  #"
+	echo "#########################"
+	echo "   Current Dir: $prjdir"
+	echo "     First Arg: $first_arg"
+	echo "Remaining Args: ${remaining_args[@]}"
+	echo ""
+	export TESTSRC=../
+	export TESTTMP=$first_arg
+	export DYLD_LIBRARY_PATH=$prjdir:$prjdir/db_plugins:$prjdir/lay_plugins
+	./ut_runner ${remaining_args[@]}
+else
+	echo "Usage: -------------------------------------------------"
+	echo "  $ macQAT2.sh <test_dir> <args to ./ut_runner>"
+	echo "    ex. $ change directory to a *.macQAT/"
+	echo "        $ macQAT2.sh  __Homebrew  tlGitTests"
+	echo "--------------------------------------------------------"
+fi
+
+#--------------
+# End of File
+#--------------