Add prevoffset to comdb2_transaction_logs

berkdb/build/db.h

@@ -597,6 +597,7 @@ struct __db_lockreq {
 #define	DB_LOG_REP_ACK		0x100   /* Send an ACK */
 #define	DB_LOG_DONT_INFLATE	0x200   /* Prevent recursive inflates */
 #define	DB_LOG_LOGICAL_COMMIT	0x400 /* This contains a logical commit */
+#define DB_LOG_NONEW		0x800 /* Don't create a new log file */
 
 #define	DB_REG_HAVE_FQLOCK	0x001   /* We already have fqlock */
 

berkdb/dbinc_auto/txn_auto.h

@@ -141,4 +141,10 @@ typedef struct __txn_ckp_recovery_args {
 #define DB___txn_regop_gen_endianize 151
 #define DB___txn_dist_prepare_endianize 152
 
+#define DB___newfile 153
+typedef struct __newfile_args {
+    u_int32_t type;
+    int32_t nextfile;
+} __newfile_args;
+
 #endif

berkdb/dbinc_auto/txn_ext.h

@@ -148,6 +148,8 @@ int __rep_commit_dist_prepared __P((DB_ENV *, const char *dist_txnid));
 int __rep_abort_dist_prepared __P((DB_ENV *, const char *dist_txnid));
 int __txn_prepared_collect_pp __P((DB_ENV *, collect_prepared_f, void *));
 int __txn_lowest_prepared_lsn __P((DB_ENV *, DB_LSN *lsn));
+int __newfile_recover __P((DB_ENV *, DBT *, DB_LSN *, db_recops, void *));
+int __newfile_log __P((DB_ENV *, DB_LSN *, int32_t, u_int32_t)); 
 
 int __txn_remevent __P((DB_ENV *, DB_TXN *, const char *, u_int8_t*));
 void __txn_remrem __P((DB_ENV *, DB_TXN *, const char *));

berkdb/log/log_put.c

@@ -272,8 +272,10 @@ __log_put_int_int(dbenv, lsnp, contextp, udbt, flags, off_context, usr_ptr)
     }
 
 
-	R_LOCK(dbenv, &dblp->reginfo);
-	lock_held = 1;
+	if (!LF_ISSET(DB_LOG_NONEW)) {
+		R_LOCK(dbenv, &dblp->reginfo);
+		lock_held = 1;
+	}
 
 	ZERO_LSN(old_lsn);
 
@@ -299,8 +301,10 @@ __log_put_int_int(dbenv, lsnp, contextp, udbt, flags, off_context, usr_ptr)
 		 * messages, but we want to drop and reacquire it a minimal
 		 * number of times.
 		 */
-		R_UNLOCK(dbenv, &dblp->reginfo);
-		lock_held = 0;
+		if (!LF_ISSET(DB_LOG_NONEW)) {
+			R_UNLOCK(dbenv, &dblp->reginfo);
+			lock_held = 0;
+		}
 		/*
 		 * If we are not a rep application, but are sharing a
 		 * master rep env, we should not be writing log records.
@@ -370,7 +374,7 @@ __log_put_int_int(dbenv, lsnp, contextp, udbt, flags, off_context, usr_ptr)
 	 * need to write out the log buffer.
 	 */
 	if (LF_ISSET(DB_FLUSH | DB_LOG_WRNOSYNC)) {
-		if (!lock_held) {
+		if (!lock_held && !LF_ISSET(DB_LOG_NONEW)) {
 			R_LOCK(dbenv, &dblp->reginfo);
 			lock_held = 1;
 		}
@@ -647,6 +651,8 @@ __log_txn_lsn(dbenv, lsnp, mbytesp, bytesp)
 	R_UNLOCK(dbenv, &dblp->reginfo);
 }
 
+int gbl_logged_newfile = 1;
+
 /*
  * __log_put_next --
  *	Put the given record as the next in the log, wherever that may
@@ -696,8 +702,9 @@ __log_put_next(dbenv, lsn, context, dbt, udbt, hdr, old_lsnp, off_context, key,
 	 * replication client environment and have been told to do so,
 	 * swap files.
 	 */
-	if (lp->lsn.offset == 0 ||
-		lp->lsn.offset + hdr->size + dbt->size > lp->log_size) {
+	if (!LF_ISSET(DB_LOG_NONEW) &&
+			(lp->lsn.offset == 0 ||
+		lp->lsn.offset + hdr->size + dbt->size > lp->log_size)) {
 		if (hdr->size + sizeof(LOGP) + dbt->size > lp->log_size) {
 			__db_err(dbenv,
 		"DB_ENV->log_put: record larger than maximum file size (%lu > %lu)",
@@ -706,6 +713,16 @@ __log_put_next(dbenv, lsn, context, dbt, udbt, hdr, old_lsnp, off_context, key,
 			return (EINVAL);
 		}
 
+		if (gbl_logged_newfile && IS_REP_MASTER(dbenv)) {
+			DB_LSN newfile_lsn;
+			ret = __newfile_log(dbenv, &newfile_lsn, lp->lsn.file + 1, DB_LOG_NONEW);
+			if (ret != 0) {
+				logmsg(LOGMSG_ERROR, "%s %d: __newfile_log failed with %d\n",
+					__func__, __LINE__, ret);
+				return (EINVAL);
+			}
+		}
+
 		if ((ret = __log_newfile(dblp, NULL)) != 0)
 			return (ret);
 

berkdb/txn/txn_auto.c

@@ -4477,6 +4477,187 @@ __txn_dist_commit_read(dbenv, recbuf, argpp)
 	return __txn_dist_commit_read_int (dbenv, recbuf, 1, argpp);
 }
 
+/*
+ * PUBLIC: int __newfile_log __P((DB_ENV *, DB_LSN *ret_lsnp, int32_t nextfile, u_int32_t flags));
+ */
+int
+__newfile_log(dbenv, ret_lsnp, nextfile, flags)
+	DB_ENV *dbenv;
+	DB_LSN *ret_lsnp;
+	int32_t nextfile;
+	u_int32_t flags;
+{
+
+	DBT logrec;
+	int ret;
+	u_int npad;
+	u_int32_t rectype;
+	u_int8_t *bp;
+
+#ifdef __txn_DEBUG
+	fprintf(stderr,"__newfile_log: begin\n");
+#endif
+
+	rectype = DB___newfile;
+	npad = 0;
+
+	if (LF_ISSET(DB_LOG_NOT_DURABLE))
+		return 0;
+
+	logrec.size = sizeof(rectype) + sizeof(nextfile);
+
+	if (CRYPTO_ON(dbenv)) {
+		npad =
+			((DB_CIPHER *)dbenv->crypto_handle)->adj_size(logrec.size);
+		logrec.size += npad;
+	}
+
+	logrec.data = alloca(logrec.size);
+	if (npad > 0)
+		memset((u_int8_t *)logrec.data + logrec.size - npad, 0, npad);
+
+	bp = logrec.data;
+
+	LOGCOPY_32(bp, &rectype);
+	bp += sizeof(rectype);
+
+	LOGCOPY_32(bp, &nextfile);
+	bp += sizeof(rectype);
+
+	ret = __log_put(dbenv,
+		ret_lsnp, (DBT *)&logrec, flags | DB_LOG_NOCOPY);
+
+	return (ret);
+}
+
+#ifdef HAVE_REPLICATION
+/* 
+ * PUBLIC: int __newfile_getpgnos __P((DB_ENV *, DBT *, DB_LSN *,
+ * PUBLIC:   db_recops, void *));
+ */
+int
+__newfile_getpgnos(dbenv, rec, lsnp, notused1, summary)
+	DB_ENV *dbenv;
+	DBT *rec;
+	DB_LSN *lsnp;
+	db_recops notused1;
+	void *summary;
+{
+	TXN_RECS *t;
+	int ret;
+	COMPQUIET(rec, NULL);
+	COMPQUIET(notused1, DB_TXN_ABORT);
+
+	t = (TXN_RECS *)summary;
+
+	if ((ret = __rep_check_alloc(dbenv, t, 1)) != 0)
+		return (ret);
+
+	t->array[t->npages].flags = LSN_PAGE_NOLOCK;
+	t->array[t->npages].lsn = *lsnp;
+	t->array[t->npages].fid = DB_LOGFILEID_INVALID;
+	memset(&t->array[t->npages].pgdesc, 0,
+		sizeof(t->array[t->npages].pgdesc));
+
+	t->npages++;
+
+	return (0);
+}
+#endif /* HAVE_REPLICATION */
+
+#ifdef HAVE_REPLICATION
+/*
+ * PUBLIC: int __newfile_getallpgnos __P((DB_ENV *, DBT *,
+ * PUBLIC:   DB_LSN *, db_recops, void *));
+ */
+
+int
+__newfile_getallpgnos(dbenv, rec, lsnp, notused1, summary)
+	DB_ENV *dbenv;
+	DBT *rec;
+	DB_LSN *lsnp;
+	db_recops notused1;
+	void *summary;
+{
+	int ret = 0;
+	COMPQUIET(notused1, DB_TXN_ABORT);
+	return (ret);
+}
+#endif /* HAVE_REPLICATION */
+
+/*
+ * PUBLIC: int __newfile_read_int __P((DB_ENV *, void *,
+ * PUBLIC:	  int do_pgswp, __newfile_args **));
+ */
+static int
+__newfile_read_int(dbenv, recbuf, do_pgswp, argpp)
+	DB_ENV *dbenv;
+	void *recbuf;
+	int do_pgswp;
+	__newfile_args **argpp;
+{
+	__newfile_args *argp;
+	u_int32_t uinttmp;
+	u_int8_t *bp;
+	int ret;
+
+	if ((ret = __os_malloc(dbenv,
+		sizeof(__newfile_args), &argp)) != 0)
+		return (ret);
+
+	bp = recbuf;
+	LOGCOPY_32(&argp->type, bp);
+	bp += sizeof(argp->type);
+
+	LOGCOPY_32(&uinttmp, bp);
+	argp->nextfile = uinttmp;
+	bp += sizeof(uinttmp);
+
+	*argpp = argp;
+	return (0);
+}
+
+int
+__newfile_print(dbenv, dbtp, lsnp, notused2, notused3)
+	DB_ENV *dbenv;
+	DBT *dbtp;
+	DB_LSN *lsnp;
+	db_recops notused2;
+	void *notused3;
+{
+	__newfile_args *argp;
+	int ret;
+
+	notused2 = DB_TXN_ABORT;
+	notused3 = NULL;
+
+	if ((ret = __newfile_read_int(dbenv, dbtp->data, 0, &argp)) != 0)
+		return (ret);
+
+	(void)printf(
+			"[%lu][%lu]__newfile: rec: %lu nextfile %lu\n",
+			(u_long)lsnp->file,
+			(u_long)lsnp->offset,
+			(u_long)argp->type,
+			(u_long)argp->nextfile);
+
+	(void)printf("\n");
+	fflush(stdout);
+	__os_free(dbenv, argp);
+
+	return (0);
+}
+
+int
+__newfile_read(dbenv, recbuf, argpp)
+	DB_ENV *dbenv;
+	void *recbuf;
+	__newfile_args **argpp;
+{
+	return __newfile_read_int (dbenv, recbuf, 1, argpp);
+}
+
+
 /*
  * PUBLIC: int __txn_init_print __P((DB_ENV *, int (***)(DB_ENV *,
  * PUBLIC:	 DBT *, DB_LSN *, db_recops, void *), size_t *));
@@ -4531,6 +4712,9 @@ __txn_init_print(dbenv, dtabp, dtabsizep)
 	if ((ret = __db_add_recovery(dbenv, dtabp, dtabsizep,
 		__txn_ckp_print, DB___txn_ckp_recovery)) != 0)
 		return (ret);
+	if ((ret = __db_add_recovery(dbenv, dtabp, dtabsizep,
+		__newfile_print, DB___newfile)) != 0)
+		return (ret);
 
 	return (0);
 }
@@ -4590,6 +4774,9 @@ __txn_init_getpgnos(dbenv, dtabp, dtabsizep)
 	if ((ret = __db_add_recovery(dbenv, dtabp, dtabsizep,
 		__txn_ckp_getpgnos, DB___txn_ckp_recovery)) != 0)
 		return (ret);
+	if ((ret = __db_add_recovery(dbenv, dtabp, dtabsizep,
+		__newfile_getpgnos, DB___newfile)) != 0)
+		return (ret);
 
 	return (0);
 }
@@ -4651,6 +4838,9 @@ __txn_init_getallpgnos(dbenv, dtabp, dtabsizep)
 	if ((ret = __db_add_recovery(dbenv, dtabp, dtabsizep,
 		__txn_ckp_getallpgnos, DB___txn_ckp_recovery)) != 0)
 		return (ret);
+	if ((ret = __db_add_recovery(dbenv, dtabp, dtabsizep,
+		__newfile_getallpgnos, DB___newfile)) != 0)
+		return (ret);
 
 	return (0);
 }
@@ -4710,5 +4900,8 @@ __txn_init_recover(dbenv, dtabp, dtabsizep)
 	if ((ret = __db_add_recovery(dbenv, dtabp, dtabsizep,
 		__txn_ckp_recover, DB___txn_ckp_recovery)) != 0)
 		return (ret);
+	if ((ret = __db_add_recovery(dbenv, dtabp, dtabsizep,
+		__newfile_recover, DB___newfile)) != 0)
+		return (ret);
 	return (0);
 }

berkdb/txn/txn_rec.c

@@ -1024,6 +1024,24 @@ err:	__os_free(dbenv, argp);
 	return (ret);
 }
 
+/*
+ * PUBLIC: int __newfile_recover
+ * PUBLIC:    __P((DB_ENV *, DBT *, DB_LSN *, db_recops, void *));
+ */
+int
+__newfile_recover(dbenv, dbtp, lsnp, op, info)
+    DB_ENV *dbenv;
+    DBT *dbtp;
+    DB_LSN *lsnp;
+    db_recops op;
+    void *info;
+{
+#ifdef DEBUG_RECOVER
+	__newfile_print(dbenv, dbtp, lsnp, op, info);
+#endif
+    return (0);
+}
+
 /*
  * PUBLIC: int __txn_ckp_recover
  * PUBLIC: __P((DB_ENV *, DBT *, DB_LSN *, db_recops, void *));

db/config.c

@@ -463,7 +463,8 @@ static char *legacy_options[] = {
     "sc_current_version 3",
     "disable_sql_table_replacement 1",
     "endianize_locklist 0",
-    "create_default_consumers_atomically 0"
+    "create_default_consumers_atomically 0",
+    "logged_newfile 0"
 };
 // clang-format on
 

db/db_tunables.c

@@ -149,6 +149,7 @@ extern int gbl_rep_skip_recovery;
 extern int gbl_retrieve_gen_from_ckp;
 extern int gbl_recovery_ckp;
 extern int gbl_reproduce_ckp_bug;
+extern int gbl_logged_newfile;
 extern int gbl_sample_queries;
 extern int gbl_sample_queries_max_queries;
 extern int gbl_slow_rep_process_txn_freq;

db/db_tunables.h

@@ -1084,6 +1084,8 @@ REGISTER_TUNABLE("recovery_ckp", "Emit a non-matchable ckp during recovery.  (De
                  &gbl_recovery_ckp, 0, NULL, NULL, NULL, NULL);
 REGISTER_TUNABLE("reproduce_ckp_bug", "Allow full-recovery ckp-gen to exceed cluster generation.  (Default: off)",
                  TUNABLE_BOOLEAN, &gbl_reproduce_ckp_bug, 0, NULL, NULL, NULL, NULL);
+REGISTER_TUNABLE("logged_newfile", "Log new-files in the transaction log. (Default: on)", TUNABLE_BOOLEAN,
+                 &gbl_logged_newfile, 0, NULL, NULL, NULL, NULL);
 REGISTER_TUNABLE("reqldiffstat", NULL, TUNABLE_INTEGER, &diffstat_thresh, READONLY, NULL, NULL, NULL, NULL);
 REGISTER_TUNABLE("reqltruncate", NULL, TUNABLE_INTEGER, &reqltruncate, READONLY,
                  NULL, NULL, NULL, NULL);