buildsys: introduce packcc as a new translator

Packcc is a compiler-compiler; it translates .peg grammar file to .c
file.  packcc was originally written by Arihiro Yoshida. Its source
repository is at sourceforge. It seems that packcc at sourceforge is
not actively maintained. Some derived repositories are at
github. Currently, our choice is https://github.com/enechaev/packcc.
It is extended to support unicode.

The source tree of packcc is grafted at misc/packcc directory.
Building packcc and ctags are integrated in the build-scripts of
Universal-ctags.

If misc/packcc directory is empty, run following command to get
source code before building ctags:
```
$ git submodule init misc/packcc
$ git submodule update misc/packcc
```

About the suffix rule about .c and .peg, @b4n and @mqudsi help me
on universal-ctags#1554.

Signed-off-by: Masatake YAMATO <yamato@redhat.com>

.gitignore

@@ -16,6 +16,7 @@
 *.TMP
 *.tmp
 *~
+/packcc
 .deps
 .dirstamp
 .DS_Store

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "misc/packcc"]
+	path = misc/packcc
+	url = https://github.com/masatake/packcc.git
+	branch = ctags

Makefile.am

@@ -12,8 +12,28 @@ EXTRA_DIST   = README.md autogen.sh \
 	       win32/ctags_vs2013.vcxproj win32/ctags_vs2013.vcxproj.filters \
 	       win32/gen-repoinfo.bat
 
+noinst_PROGRAMS = packcc
 bin_PROGRAMS = ctags
 
+packcc_CPPFLAGS =
+packcc_CFLAGS =
+dist_packcc_SOURCES = $(PACKCC_SRCS)
+
+if HAVE_STRNLEN
+packcc_CPPFLAGS += -DUSE_SYSTEM_STRNLEN
+endif
+
+if USE_NO_FORMATTER_FOR_SIZE_T
+packcc_CPPFLAGS += -DSIZE_T_FMT_CHAR='""'
+endif
+if USE_ULONG_FORMATTER_FOR_SIZE_T
+packcc_CPPFLAGS += -DSIZE_T_FMT_CHAR='"l"'
+endif
+if USE_ULONGLONG_FORMATTER_FOR_SIZE_T
+packcc_CPPFLAGS += -DSIZE_T_FMT_CHAR='"ll"'
+endif
+
+
 if USE_READCMD
 bin_PROGRAMS+= readtags
 readtags_CPPFLAGS = -I. -I$(srcdir) -I$(srcdir)/main -I$(srcdir)/read
@@ -48,10 +68,16 @@ PARSER_SRCS += $(YAML_SRCS)
 PARSER_HEADS += $(YAML_HEADS)
 endif
 
-ctags_CPPFLAGS = -I. -I$(srcdir) -I$(srcdir)/main
+PARSER_SRCS += $(PEG_SRCS)
+PARSER_HEADS += $(PEG_HEADS) $(PEG_EXTRA_HEADS)
+
+ctags_CPPFLAGS = -I. -I$(srcdir) -I$(srcdir)/main -DHAVE_PACKCC
 if ENABLE_DEBUGGING
 ctags_CPPFLAGS+= $(DEBUG_CPPFLAGS)
 endif
+if HAVE_STRNLEN
+ctags_CPPFLAGS += -DUSE_SYSTEM_STRNLEN
+endif
 ctags_CPPFLAGS+= $(FNMATCH_CPPFLAGS)
 ctags_CPPFLAGS+= $(REGCOMP_CPPFLAGS)
 ctags_CPPFLAGS+= -DHAVE_REPOINFO_H
@@ -104,6 +130,19 @@ SUFFIXES += .ctags
 	$(optlib2c_verbose)$(OPTLIB2C) $< > $@
 $(OPTLIB2C_INPUT): $(OPTLIB2C) Makefile
 endif
+
+packcc_verbose = $(packcc_verbose_@AM_V@)
+packcc_verbose_ = $(packcc_verbose_@AM_DEFAULT_V@)
+packcc_verbose_0 = @echo PACKCC "    $@";
+PACKCC = $(srcdir)/packcc$(EXEEXT)
+SUFFIXES += .peg
+.peg.c:
+	$(packcc_verbose)$(PACKCC) $<
+.peg.h:
+	$(packcc_verbose)$(PACKCC) $<
+# You cannot use $(PACKCC) as a target name here.
+$(PEG_INPUT): packcc$(EXEEXT) Makefile
+$(PEG_HEADS): packcc$(EXEEXT) Makefile
 dist_ctags_SOURCES = $(ALL_HEADS) $(ALL_SRCS)
 
 if INSTALL_ETAGS

README.md

@@ -14,6 +14,27 @@ The goal of the project is preparing and maintaining common/unified working
 space where people interested in making ctags better can work
 together.
 
+## Getting PACKCC compiler-compiler ##
+
+Packcc is a compiler-compiler; it translates .peg grammar file to .c
+file.  packcc was originally written by Arihiro Yoshida. Its source
+repository is at sourceforge. It seems that packcc at sourceforge is
+not actively maintained. Some derived repositories are at
+github. Currently, our choice is
+https://github.com/enechaev/packcc. It is the most active one in the
+derived repositories.
+
+The source tree of packcc is grafted at misc/packcc directory.
+Building packcc and ctags are integrated in the build-scripts of
+Universal-ctags.
+
+If misc/packcc directory is empty, run following command to get
+source code before building ctags:
+```
+$ git submodule init misc/packcc
+$ git submodule update misc/packcc
+```
+
 ## The latest build and package ##
 
 If you want to try the latest universal-ctags without building it yourself...

autogen.sh

@@ -5,6 +5,11 @@ set -xe
 type autoreconf || exit 1
 type pkg-config || exit 1
 
+if ! [ -f misc/packcc/packcc.c ]; then
+	git submodule init
+	git submodule update
+fi
+
 if [ -z "${MAKE}" ]; then
 	if type make > /dev/null; then
 		MAKE=make

configure.ac

@@ -363,8 +363,8 @@ CHECK_HEADER_DEFINE(INT_MAX, [limits.h],,
 		AC_DEFINE(INT_MAX, MAXINT), AC_DEFINE(INT_MAX, 32767)))
 
 
-# Checks for typedefs
-# -------------------
+# Checks for typedefs and types
+# -----------------------------
 
 AC_TYPE_SIZE_T
 AC_TYPE_OFF_T
@@ -400,6 +400,41 @@ AC_CHECK_HEADERS([stdbool.h],
 	AC_DEFINE([false], [0])
 ])
 
+AC_MSG_CHECKING(if "%zu" in printf works)
+AC_RUN_IFELSE([AC_LANG_SOURCE([[
+#include <stdio.h>
+int main(void) {
+	static char buffer[10];
+	size_t i = 999;
+	sprintf(buffer, "%zu", i);
+	return !(buffer[0] == '9' && buffer[1] == '9' && buffer[2] == '9');
+}
+]])],[have_zu_formatter=yes],[have_zu_formatter=no])
+AC_MSG_RESULT($have_zu_formatter)
+
+if test x$have_zu_formatter != xyes; then
+	AC_CHECK_SIZEOF(size_t)
+	AC_CHECK_SIZEOF(unsigned int)
+	AC_CHECK_SIZEOF(unsigned long)
+	AC_CHECK_SIZEOF(unsigned long long)
+	AC_MSG_CHECKING(alternative for "%zu")
+	if test x"$ac_cv_sizeof_size_t" = x"$ac_cv_sizeof_unsigned_int"; then
+		use_no_formatter_for_size_t=yes
+		AC_MSG_RESULT("%u")
+	elif test x"$ac_cv_sizeof_size_t" = x"$ac_cv_sizeof_unsigned_long"; then
+		use_ulong_formatter_for_size_t=yes
+		AC_MSG_RESULT("%lu")
+	elif test x"$ac_cv_sizeof_size_t" = x"$ac_cv_sizeof_unsigned_long_long"; then
+		use_ulonglong_formatter_for_size_t=yes
+		AC_MSG_RESULT("%llu")
+	else
+		AC_MSG_ERROR(no alternative for "%zu" formatter)
+	fi
+fi
+
+AM_CONDITIONAL([USE_NO_FORMATTER_FOR_SIZE_T], [test "x${use_no_formatter_for_size_t}" = "xyes"])
+AM_CONDITIONAL([USE_ULONG_FORMATTER_FOR_SIZE_T], [test "x${use_ulong_formatter_for_size_t}" = "xyes"])
+AM_CONDITIONAL([USE_ULONGLONG_FORMATTER_FOR_SIZE_T], [test "x${use_ulonglong_formatter_for_size_t}" = "xyes"])
 
 # Checks for compiler characteristics
 # -----------------------------------
@@ -714,6 +749,10 @@ AC_CHECK_DECLS([__environ], [], [], [
 AC_CHECK_DECLS([_NSGetEnviron],[],[],[
 [#include <crt_externs.h>]
 ])
+AC_CHECK_DECLS([strnlen],[have_strnlen=yes],[],[
+[#include <string.h>]
+])
+AM_CONDITIONAL(HAVE_STRNLEN, test "x$have_strnlen" = xyes)
 
 if test "$CTAGS_NAME_EXECUTABLE" != ctags ; then
 	AC_MSG_NOTICE(Changing name of 'ctags' for $CTAGS_NAME_EXECUTABLE)

main/options.c

@@ -562,6 +562,10 @@ static struct Feature {
 #endif
 #ifdef HAVE_ASPELL
 	{"aspell", "linked with code for spell checking (internal use)"},
+#endif
+#ifdef HAVE_PACKCC
+	/* The test harnesses use this as hints for skipping test cases */
+	{"packcc", "has peg based parser(s)"},
 #endif
 	{NULL,}
 };

makefiles/peg_input.mak

@@ -0,0 +1,4 @@
+# -*- makefile -*-
+PEG_INPUT = \
+       \
+       $(NULL)

misc/travis-check.sh

@@ -52,7 +52,7 @@ if [ "$TARGET" = "Unix" ]; then
         make -j2 check roundtrip TRAVIS=1
     fi
 elif [ "$TARGET" = "Mingw32" ]; then
-    make -j2 CC=i686-w64-mingw32-gcc -f mk_mingw.mak
+    make -j2 CC=i686-w64-mingw32-gcc CC_FOR_PACKCC=gcc -f mk_mingw.mak
     # Don't run test units in Mingw32 target
 else
     echo 'Invalid $TARGET value' 1>&2

mk_mingw.mak

@@ -5,14 +5,19 @@ include source.mak
 REGEX_DEFINES = -DHAVE_REGCOMP -D__USE_GNU -Dbool=int -Dfalse=0 -Dtrue=1 -Dstrcasecmp=stricmp
 
 CFLAGS = -Wall -std=gnu99
-DEFINES = -DWIN32 $(REGEX_DEFINES)
+# sizeof (size_t) == sizeof(unsigned long) == 4 on i686-w64-mingw32-gcc.
+SIZE_T_FMT_CHAR='""'
+COMMON_DEFINES=-DUSE_SYSTEM_STRNLEN
+DEFINES = -DWIN32 $(REGEX_DEFINES) -DHAVE_PACKCC $(COMMON_DEFINES)
 INCLUDES = -I. -Imain -Ignu_regex -Ifnmatch -Iparsers
 CC = gcc
 OPTLIB2C = ./misc/optlib2c
+PACKCC   = ./packcc.exe
 OBJEXT = o
 ALL_OBJS += $(REGEX_OBJS)
 ALL_OBJS += $(FNMATCH_OBJS)
 ALL_OBJS += $(WIN32_OBJS)
+ALL_OBJS += $(PEG_OBJS)
 VPATH = . ./main ./parsers ./optlib ./read ./win32
 
 ifeq (yes, $(WITH_ICONV))
@@ -28,14 +33,15 @@ OPT = -O4 -Os -fexpensive-optimizations
 LDFLAGS = -s
 endif
 
-.SUFFIXES: .c .o .ctags
+.SUFFIXES: .c .o .ctags .peg
 
 #
 # Silent/verbose commands
 #
 # when V is not set the output of commands is omitted or simplified
 #
 V	 ?= 0
+CC_FOR_PACKCC ?= $(CC)
 
 SILENT   = $(SILENT_$(V))
 SILENT_0 = @
@@ -49,18 +55,35 @@ V_OPTLIB2C   = $(V_OPTLIB2C_$(V))
 V_OPTLIB2C_0 = @echo [OPTLIB2C] $@;
 V_OPTLIB2C_1 =
 
+V_PACKCC   = $(V_PACKCC_$(V))
+V_PACKCC_0 = @echo [PACKCC] $@;
+V_PACKCC_1 =
+
 
 .c.o:
 	$(V_CC) $(CC) -c $(OPT) $(CFLAGS) $(DEFINES) $(INCLUDES) -o $@ $<
 
 %.c: %.ctags $(OPTLIB2C)
 	$(V_OPTLIB2C) $(OPTLIB2C) $< > $@
 
-all: ctags.exe readtags.exe
+peg/%.c peg/%.h: peg/%.peg $(PACKCC)
+	$(V_PACKCC) $(PACKCC) $<
+
+all: $(PACKCC) ctags.exe readtags.exe
 
 ctags: ctags.exe
 
-ctags.exe: $(ALL_OBJS) $(ALL_HEADS) $(REGEX_HEADS) $(FNMATCH_HEADS) $(WIN32_HEADS)
+$(PACKCC_SRCS):
+	git submodule init
+	git submodule update
+
+$(PACKCC_OBJS): $(PACKCC_SRCS)
+	$(V_CC) $(CC_FOR_PACKCC) -c $(OPT) $(CFLAGS) $(COMMON_DEFINES) -DSIZE_T_FMT_CHAR=$(SIZE_T_FMT_CHAR) -o $@ $<
+
+$(PACKCC): $(PACKCC_OBJS)
+	$(V_CC) $(CC_FOR_PACKCC) $(OPT) -o $@ $^
+
+ctags.exe: $(ALL_OBJS) $(ALL_HEADS) $(PEG_HEADS) $(PEG_EXTRA_HEADS) $(REGEX_HEADS) $(FNMATCH_HEADS) $(WIN32_HEADS)
 	$(V_CC) $(CC) $(OPT) $(CFLAGS) $(LDFLAGS) $(DEFINES) $(INCLUDES) -o $@ $(ALL_OBJS) $(LIBS)
 
 read/%.o: read/%.c

source.mak

@@ -140,6 +140,12 @@ MAIN_SRCS =				\
 include makefiles/optlib2c_input.mak
 OPTLIB2C_SRCS = $(OPTLIB2C_INPUT:.ctags=.c)
 
+include makefiles/peg_input.mak
+PEG_SRCS = $(PEG_INPUT:.peg=.c)
+PEG_HEADS = $(PEG_INPUT:.peg=.h)
+PEG_EXTRA_HEADS = $(PEG_INPUT:.peg=_pre.h) $(PEG_INPUT:.peg=_post.h)
+PEG_OBJS = $(PEG_SRCS:.c=.$(OBJEXT))
+
 PARSER_HEADS = \
 	parsers/cxx/cxx_debug.h \
 	parsers/cxx/cxx_keyword.h \
@@ -329,4 +335,11 @@ READTAGS_SRCS  = \
 READTAGS_HEADS = read/readtags.h
 READTAGS_OBJS  = $(READTAGS_SRCS:.c=.$(OBJEXT))
 
+PACKCC_SRCS = \
+	    misc/packcc/packcc.c \
+	    \
+	    $(NULL)
+
+PACKCC_OBJS = $(PACKCC_SRCS:.c=.$(OBJEXT))
+
 # vim: ts=8