Update upstream source from tag 'upstream/buster/4.99.24_rc2-buster'

Update to upstream version '4.99.24~rc2-buster'
with Debian dir 9b64f9ccc95db73066f8ffe9f9be722f4557d80c

configure.ac

@@ -35,7 +35,7 @@
 
 AC_PREREQ([2.68])
 
-AC_INIT([NaviServer],[4.99.24rc1],[naviserver-devel@lists.sourceforge.net])
+AC_INIT([NaviServer],[4.99.24rc2],[naviserver-devel@lists.sourceforge.net])
 AC_CONFIG_MACRO_DIR([m4])
 
 #AM_INIT_AUTOMAKE([1.9 -Wall -Werror foreign])

doc/src/naviserver/ns_cookie.man

@@ -83,6 +83,7 @@ recommended to use a unique name for each restricted cookie.
      [opt [option "-domain [arg d]"]] \
      [opt [option "-path [arg p]"]] \
      [opt [option "-replace [arg bool]"]] \
+     [opt [option "-samesite [arg t]"]] \
      [opt [option "-secure [arg bool]"]] \
      [opt [option --]] \
      [arg name]  ]
@@ -132,8 +133,9 @@ The default value is false.
 [opt_def -samesite [arg strict|lax|none]]
 When the flag is set it prevents the browser from
 sending this cookie along with cross-site requests to mitigate cross site
-scripting attacks. Permissible values are [term strict], [term lax],
-or [term none] (default). While the value [term strict] prevents
+scripting attacks. Permissible values are [term strict], [term lax] (default),
+or [term none] (only allowed together with [option -secure]).
+While the value [term strict] prevents
 sending the cookie to the target site in all cross-site browsing
 context, the value of [term lax] allows sending the cookie when the
 user clicks on regular links. For details, see

doc/src/naviserver/ns_getform.man

@@ -40,12 +40,13 @@ when the connection is closed.
 [para]
 
 In case in the provided form data contains invalid UTF-8 text, an
-exception is raised. When the global or per-server configuration
-variable [term formfallbackcharset] is provided, a second decoding
-attempt is made with this charset and a warning is generated when
-invalid UTF-8 data is encountered.  Note that the second attempt using
-the fallback charset is made as well, when an invalid charset was
-provided with in the content-type header field.
+exception is raised (with exception code NS_INVALID_UTF8). When the
+global or per-server configuration variable [term formfallbackcharset]
+is provided, a second decoding attempt is made with this charset and a
+warning is generated when invalid UTF-8 data is encountered.  Note
+that the second attempt using the fallback charset is made as well,
+when an invalid charset was provided with in the content-type header
+field.
 
 By using the option [option "-fallbackcharset [arg charset]"]
 the fallback charset can be specified explicitly, which overrides

doc/src/naviserver/ns_parsequery.man

@@ -20,10 +20,11 @@ Parse HTTP query parameters.
 
 [list_end]
 
-This command parses the specified HTTP [arg querystring]
-into an [cmd ns_set], which is returned.  The keys and values will be decoded
-based by the rules used for [cmd ns_urlencode]. In case the
-query cannot be parsed to the charset, an exception is raised.
+This command parses the specified HTTP [arg querystring] into an
+[cmd ns_set], which is returned.  The keys and values will be decoded based
+by the rules used for [cmd ns_urlencode]. In case the query cannot be
+parsed to the charset, an exception is raised (with exception code
+NS_INVALID_UTF8).
 
 [para]
 The option [option -charset] can be used to specify the character set of
@@ -32,7 +33,7 @@ the encode operation.
 [para]
 
 The option [option -fallbackcharset] can be used to specify an
-alternative character set in cases, the conversion of the
+alternative character set when the conversion of the
 [arg querystring] leads to invalid UTF-8. Instead of raising an exception,
 a second attempt of decoding the query string is made with the fallback
 charset. Note that the fallback charset can be provided as well via the

doc/src/naviserver/ns_urldecode.man

@@ -16,19 +16,37 @@
 
  [call [cmd ns_urldecode] \
      [opt [option "-charset [arg charset]"]] \
+     [opt [option "-fallbackcharset [arg fallbackcharset]"]] \
      [opt [option "-part [arg part]"]] \
+     [opt [arg "--"]] \
      [arg string]]
 
 [list_end]
 
 The option [option -charset] can be used to specify the character set of
-the decode operation. Option [option -part] is used to specify the
-[arg query] (default) or [arg path] encoding. In case the string
+the decode operation. In case the string
 cannot be decoded into the specified charset (on most platforms UTF-8)
 an exception is raised.
 
-[see_also nsd ns_urlencode ns_parsequery ns_charsets ns_urlcharset]
-[keywords "global built-in" encoding charset url]
+[para]
+
+In cases, where no explicit [option "-charset"] is specified, the
+option [option -fallbackcharset] can be used to specify an alternative
+character set when the conversion of the [arg querystring]
+leads to invalid UTF-8. Instead of raising an exception, a second
+attempt of decoding the query string is made with the fallback
+charset. In contrary to [cmd ns_getform]
+and [cmd ns_parsequery] the configuration parameter
+is not used [term formfallbackcharset], since this happens outside
+automatic form processing.
+
+[para]
+
+The option [option -part] is used to specify the
+[arg query] (default) or [arg path] encoding.
+
+[see_also nsd ns_urlencode ns_parsequery ns_charsets ns_urlcharset ns_getform]
+[keywords "global built-in" encoding charset URL formfallbackcharset]
 
 [manpage_end]
 

doc/src/naviserver/ns_valid_utf8.man

@@ -11,11 +11,15 @@
 [list_begin definitions]
 
 [call [cmd ns_valid_utf8] \
-        [arg string]]
+        [arg string] \
+        [opt [arg varname]] \
+        ]
 
 [para] This function checks whether the provided argument (a Tcl
 byte-array) contains a valid UTF-8 code). The function returns a
-boolean result.
+boolean result. When the optional [arg varname] is specified,
+a variable with this name is set in failure cases to the
+invalid part of the input [arg string].
 
 [para] Note that this function
 only makes sense when applied on binary data, typically read from a
@@ -27,6 +31,11 @@ socket or a file.
 
  % ns_valid_utf8 "foo\x85"
  0
+
+ % ns_valid_utf8 mot\x85rhead errorString
+ 0
+ % set errorString
+ mot|\x85r|...
 [example_end]
 
 [list_end]

include/ns.h

@@ -3080,7 +3080,7 @@ Ns_GetBinaryString(Tcl_Obj *obj, bool forceBinary, int *lengthPtr, Tcl_DString *
     NS_GNUC_NONNULL(1) NS_GNUC_NONNULL(3) NS_GNUC_NONNULL(4);
 
 NS_EXTERN bool
-Ns_Valid_UTF8(const unsigned char *bytes, size_t nrBytes)
+Ns_Valid_UTF8(const unsigned char *bytes, size_t nrBytes, Tcl_DString *dsPtr)
     NS_GNUC_NONNULL(1);
 
 NS_EXTERN bool
@@ -3391,7 +3391,7 @@ Ns_UrlQueryEncode(Ns_DString *dsPtr, const char *urlSegment, Tcl_Encoding encodi
     NS_GNUC_NONNULL(1) NS_GNUC_NONNULL(2);
 
 NS_EXTERN char *
-Ns_UrlQueryDecode(Ns_DString *dsPtr, const char *urlSegment, Tcl_Encoding encoding, int *resultPtr)
+Ns_UrlQueryDecode(Ns_DString *dsPtr, const char *urlSegment, Tcl_Encoding encoding, Ns_ReturnCode *resultPtr)
     NS_GNUC_NONNULL(1) NS_GNUC_NONNULL(2);
 
 NS_EXTERN char *

nsd/cookies.c

@@ -63,6 +63,14 @@ static CookieParser GetFromSetCookieHeader;
 static char *CopyCookieValue(Tcl_DString *dest, char *valueStart)
     NS_GNUC_NONNULL(1) NS_GNUC_NONNULL(2);
 
+static Ns_ObjvTable samesiteValues[] = {
+    {"strict", UCHAR('s')},
+    {"lax",    UCHAR('l')},
+    {"none",   UCHAR('n')},
+    {NULL,    0u}
+};
+
+
 
 /*
  *----------------------------------------------------------------------
@@ -614,14 +622,8 @@ NsTclSetCookieObjCmd(ClientData UNUSED(clientData), Tcl_Interp *interp,
     Ns_Conn       *conn;
     char          *name, *data, *domain = NULL, *path = NULL;
     int            secure = 0, scriptable = 0, discard = 0, replace = 0, result;
-    int            samesite = INTCHAR('I');
+    int            samesite = INTCHAR('l');
     Ns_Time       *expiresPtr = NULL;
-    static Ns_ObjvTable samesiteValues[] = {
-        {"strict", UCHAR('s')},
-        {"lax",    UCHAR('l')},
-        {"none",   UCHAR('n')},
-        {NULL,    0u}
-    };
     Ns_ObjvSpec    opts[] = {
         {"-discard",    Ns_ObjvBool,   &discard,    NULL},
         {"-domain",     Ns_ObjvString, &domain,     NULL},
@@ -660,11 +662,20 @@ NsTclSetCookieObjCmd(ClientData UNUSED(clientData), Tcl_Interp *interp,
         if (replace != 0) {
             flags |= NS_COOKIE_REPLACE;
         }
-        if (samesite == INTCHAR('s') || samesite == INTCHAR('S')) {
+        /*
+         * If "-samesite none" flag was provided, and secure was not set, fall
+         * back to "-samesite lax" and complain.
+         */
+        if (samesite == INTCHAR('n') && secure == 0) {
+            Ns_Log(Warning, "cookie '%s': trying to set '-samesite none' "
+                   "without the '-secure' flag. Fall back to -samesite lax", name);
+            samesite = INTCHAR('l');
+        }
+        if (samesite == INTCHAR('s')) {
             flags |= NS_COOKIE_SAMESITE_STRICT;
-        } else if (samesite == INTCHAR('l') || samesite == INTCHAR('L')) {
+        } else if (samesite == INTCHAR('l')) {
             flags |= NS_COOKIE_SAMESITE_LAX;
-        } else if (samesite == INTCHAR('n') || samesite == INTCHAR('N')) {
+        } else if (samesite == INTCHAR('n')) {
             flags |= NS_COOKIE_SAMESITE_NONE;
         }
 
@@ -804,12 +815,14 @@ NsTclDeleteCookieObjCmd(ClientData UNUSED(clientData), Tcl_Interp *interp,
     Ns_Conn        *conn;
     char           *name, *domain = NULL, *path = NULL;
     int             secure = 0, replace = 0, result;
+    int             samesite = INTCHAR('l');
     Ns_ObjvSpec     opts[] = {
-        {"-secure",  Ns_ObjvBool,   &secure,  NULL},
-        {"-domain",  Ns_ObjvString, &domain,  NULL},
-        {"-path",    Ns_ObjvString, &path,    NULL},
-        {"-replace", Ns_ObjvBool,   &replace, NULL},
-        {"--",       Ns_ObjvBreak,  NULL,     NULL},
+        {"-domain",  Ns_ObjvString, &domain,   NULL},
+        {"-path",    Ns_ObjvString, &path,     NULL},
+        {"-replace", Ns_ObjvBool,   &replace,  NULL},
+        {"-samesite",Ns_ObjvIndex,  &samesite, samesiteValues},
+        {"-secure",  Ns_ObjvBool,   &secure,   NULL},
+        {"--",       Ns_ObjvBreak,  NULL,      NULL},
         {NULL, NULL, NULL, NULL}
     };
     Ns_ObjvSpec args[] = {
@@ -831,6 +844,24 @@ NsTclDeleteCookieObjCmd(ClientData UNUSED(clientData), Tcl_Interp *interp,
             flags |= NS_COOKIE_SECURE;
         }
 
+        /*
+         * If "-samesite none" flag was provided, and secure was not set, fall
+         * back to "-samesite lax" and complain.
+         */
+        if (samesite == INTCHAR('n') && secure == 0) {
+            Ns_Log(Warning, "cookie '%s': trying to set '-samesite none' "
+                   "without the '-secure' flag. Fall back to -samesite lax", name);
+            samesite = INTCHAR('l');
+        }
+
+        if (samesite == INTCHAR('s')) {
+            flags |= NS_COOKIE_SAMESITE_STRICT;
+        } else if (samesite == INTCHAR('l')) {
+            flags |= NS_COOKIE_SAMESITE_LAX;
+        } else if (samesite == INTCHAR('n')) {
+            flags |= NS_COOKIE_SAMESITE_NONE;
+        }
+
         Ns_ConnSetCookieEx(conn, name, NULL, (time_t)0, domain, path,
                            NS_COOKIE_EXPIRENOW|flags);
         result = TCL_OK;

nsd/driver.c

@@ -4848,7 +4848,7 @@ SockSetServer(Sock *sockPtr)
         && reqPtr->request.url != NULL
         ) {
         if (!Ns_Valid_UTF8((const unsigned char *)reqPtr->request.url,
-                           strlen(reqPtr->request.url))) {
+                           strlen(reqPtr->request.url), NULL)) {
             Ns_Log(Warning, "Invalid UTF-8 encoding in url '%s'",
                    reqPtr->request.url);
             bad_request = NS_TRUE;

nsd/encoding.c

@@ -746,6 +746,72 @@ AddCharset(const char *charset, const char *name)
     Ns_DStringFree(&ds);
 }
 
+/*
+ *----------------------------------------------------------------------
+ *
+ * NsGetFallbackEncoding --
+ *
+ *      Obtain a fallback encoding either from a specified argument
+ *      (fallbackCharsetObj) or from the configuration.
+ *
+ *      The resolving order is
+ *        - use command line parameter, if specified.
+ *        - use per server parameter "formFallbackCharset" if specified;
+ *        - use global server parameter "formFallbackCharset" if specified.
+ *      The last two options are only tried, when "fromConfig" is true.
+ *
+ * Results:
+ *      NS_OK or NS_ERROR. In the error case, an error message is left in the
+ *      interp result.
+ *
+ * Side effects:
+ *      None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Ns_ReturnCode
+NsGetFallbackEncoding(Tcl_Interp *interp, NsServer *servPtr,
+                      Tcl_Obj *fallbackCharsetObj, bool fromConfig,
+                      Tcl_Encoding *encodingPtr)
+{
+    const char   *fallbackCharsetString = NULL;
+    Ns_ReturnCode result = NS_OK;
+
+    NS_NONNULL_ASSERT(interp != NULL);
+    NS_NONNULL_ASSERT(encodingPtr != NULL);
+
+    if (fallbackCharsetObj != NULL) {
+        fallbackCharsetString = Tcl_GetString(fallbackCharsetObj);
+        if (*fallbackCharsetString == '\0') {
+            fallbackCharsetString = NULL;
+        }
+    }
+    if (fromConfig) {
+        if (fallbackCharsetString == NULL && servPtr != NULL) {
+            fallbackCharsetString = servPtr->encoding.formFallbackCharset;
+        }
+        if (fallbackCharsetString == NULL && servPtr != NULL) {
+            fallbackCharsetString = nsconf.formFallbackCharset;
+        }
+    }
+    Ns_Log(Debug, "NsGetFallbackEncoding fromConfig %p %d, '%s'",
+           (void*)fallbackCharsetObj, fromConfig, fallbackCharsetString);
+    if (fallbackCharsetString != NULL) {
+        *encodingPtr = Ns_GetCharsetEncoding(fallbackCharsetString);
+        if (*encodingPtr == NULL) {
+            Ns_TclPrintfResult(interp,
+                               "invalid fallback encoding: '%s'",
+                               fallbackCharsetString);
+            result = NS_ERROR;
+        }
+    } else {
+        *encodingPtr = NULL;
+    }
+
+    return result;
+}
+
 /*
  * Local Variables:
  * mode: c