diff --git a/src/packages/database/package-lock.json b/src/packages/database/package-lock.json
index d81ed97cee..7878c5d3c5 100644
--- a/src/packages/database/package-lock.json
+++ b/src/packages/database/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@cocalc/database",
- "version": "0.17.1",
+ "version": "0.17.2",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@cocalc/database",
- "version": "0.17.1",
+ "version": "0.17.2",
"license": "SEE LICENSE.md",
"workspaces": [
"./",
diff --git a/src/packages/database/package.json b/src/packages/database/package.json
index bcfc5e4595..89e6450353 100644
--- a/src/packages/database/package.json
+++ b/src/packages/database/package.json
@@ -1,6 +1,6 @@
{
"name": "@cocalc/database",
- "version": "0.17.1",
+ "version": "0.17.2",
"description": "CoCalc: code for working with our PostgreSQL database",
"exports": {
".": "./dist/index.js",
@@ -19,8 +19,8 @@
],
"dependencies": {
"@cocalc/backend": "^1.12.1",
- "@cocalc/server": "^0.19.0",
- "@cocalc/util": "^1.32.1",
+ "@cocalc/server": "^0.20.0",
+ "@cocalc/util": "^1.32.2",
"@types/lodash": "^4.14.176",
"@types/lru-cache": "^5.1.1",
"@types/pg": "^8.6.1",
diff --git a/src/packages/database/postgres-blobs.coffee b/src/packages/database/postgres-blobs.coffee
index 9a18b7d3b2..97477ca4ad 100644
--- a/src/packages/database/postgres-blobs.coffee
+++ b/src/packages/database/postgres-blobs.coffee
@@ -609,7 +609,7 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
dbg("determine inactive syncstring ids")
@_query
query : 'SELECT string_id FROM syncstrings'
- where : [{'last_active <= $::TIMESTAMP' : misc.days_ago(opts.age_days)}, 'archived IS NULL']
+ where : [{'last_active <= $::TIMESTAMP' : misc.days_ago(opts.age_days)}, 'archived IS NULL', 'huge IS NOT TRUE']
limit : opts.limit
timeout_s : TIMEOUT_LONG_S
cb : all_results 'string_id', (err, v) =>
@@ -641,6 +641,9 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
# Offlines and archives the patch, unless the string is active very recently, in
# which case this is a no-op.
+ #
+ # TODO: this ignores all syncstrings marked as "huge:true", because the patches are too large.
+ # come up with a better strategy (incremental?) to generate the blobs to avoid the problem.
archive_patches: (opts) =>
opts = defaults opts,
string_id : required
@@ -649,13 +652,13 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
cutoff : misc.minutes_ago(30) # never touch anything this new
cb : undefined
dbg = @_dbg("archive_patches(string_id='#{opts.string_id}')")
- syncstring = patches = blob_uuid = project_id = last_active = undefined
+ syncstring = patches = blob_uuid = project_id = last_active = huge = undefined
where = {"string_id = $::CHAR(40)" : opts.string_id}
async.series([
(cb) =>
- dbg("get project_id")
+ dbg("get syncstring info")
@_query
- query : "SELECT project_id, archived, last_active FROM syncstrings"
+ query : "SELECT project_id, archived, last_active, huge FROM syncstrings"
where : where
cb : one_result (err, x) =>
if err
@@ -667,11 +670,16 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
else
project_id = x.project_id
last_active = x.last_active
+ huge = !!x.huge
+ dbg("got last_active=#{last_active} project_id=#{project_id} huge=#{huge}")
cb()
(cb) =>
if last_active? and last_active >= opts.cutoff
dbg("excluding due to cutoff")
cb(); return
+ if huge
+ dbg("excluding due to being huge")
+ cb(); return
dbg("get patches")
@export_patches
string_id : opts.string_id
@@ -681,6 +689,8 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
(cb) =>
if last_active? and last_active >= opts.cutoff
cb(); return
+ if huge
+ cb(); return
dbg("create blob from patches")
try
blob = Buffer.from(JSON.stringify(patches))
@@ -689,20 +699,30 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
# need to break patches up...
# This is not exactly the end of the world as the entire point of all this is to
# just save some space in the database...
- cb(err)
- return
- dbg('save blob')
- blob_uuid = misc_node.uuidsha1(blob)
- @save_blob
- uuid : blob_uuid
- blob : blob
- project_id : project_id
- compress : opts.compress
- level : opts.level
- cb : cb
+ dbg('error creating blob, marking syncstring as being "huge": ' + err)
+ huge = true
+ @_query
+ query : "UPDATE syncstrings"
+ set : {huge : true}
+ where : where
+ cb : (err) =>
+ cb(err)
+ return
+ if not huge
+ dbg('save blob')
+ blob_uuid = misc_node.uuidsha1(blob)
+ @save_blob
+ uuid : blob_uuid
+ blob : blob
+ project_id : project_id
+ compress : opts.compress
+ level : opts.level
+ cb : cb
(cb) =>
if last_active? and last_active >= opts.cutoff
cb(); return
+ if huge
+ cb(); return
dbg("update syncstring to indicate patches have been archived in a blob")
@_query
query : "UPDATE syncstrings"
@@ -712,6 +732,8 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
(cb) =>
if last_active? and last_active >= opts.cutoff
cb(); return
+ if huge
+ cb(); return
dbg("actually deleting patches")
delete_patches(db:@, string_id: opts.string_id, cb:cb)
], (err) => opts.cb?(err))
@@ -789,7 +811,12 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
opts.cb(err)
else
for p in patches
- p.time = new Date(p.epoch)
+ # TODO why using epoch and then converting to Date, why not just taking time?
+ # Besides that: @hsy noticed in development that p.epoch could be a string, resulting in an invalid date.
+ if typeof p.epoch == 'string'
+ p.time = new Date(parseInt(p.epoch))
+ else
+ p.time = new Date(p.epoch)
delete p.epoch
opts.cb(undefined, patches)
diff --git a/src/packages/hub/package-lock.json b/src/packages/hub/package-lock.json
index cb31a081e8..604273db1c 100644
--- a/src/packages/hub/package-lock.json
+++ b/src/packages/hub/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@cocalc/hub",
- "version": "1.51.0",
+ "version": "1.51.1",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@cocalc/hub",
- "version": "1.51.0",
+ "version": "1.51.1",
"license": "SEE LICENSE.md",
"workspaces": [
"./",
diff --git a/src/packages/hub/package.json b/src/packages/hub/package.json
index eb34e89e5d..3e469bb057 100644
--- a/src/packages/hub/package.json
+++ b/src/packages/hub/package.json
@@ -1,6 +1,6 @@
{
"name": "@cocalc/hub",
- "version": "1.51.0",
+ "version": "1.51.1",
"description": "CoCalc: Backend webserver component",
"exports": {
"./*": "./dist/*.js",
@@ -22,12 +22,12 @@
"@cocalc/assets": "^1.6.0",
"@cocalc/backend": "^1.12.1",
"@cocalc/cdn": "^1.10.0",
- "@cocalc/database": "^0.17.1",
+ "@cocalc/database": "^0.17.2",
"@cocalc/frontend": "^1.41.0",
- "@cocalc/next": "^0.39.0",
+ "@cocalc/next": "^0.39.1",
"@cocalc/server": "^0.20.0",
"@cocalc/static": "^1.74.0",
- "@cocalc/util": "^1.32.1",
+ "@cocalc/util": "^1.32.2",
"@passport-next/passport-google-oauth2": "^1.0.0",
"@passport-next/passport-oauth2": "^2.1.1",
"@sendgrid/client": "^7.4.6",
diff --git a/src/packages/next/package-lock.json b/src/packages/next/package-lock.json
index 34a0192ff3..f219ee4cad 100644
--- a/src/packages/next/package-lock.json
+++ b/src/packages/next/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@cocalc/next",
- "version": "0.39.0",
+ "version": "0.39.1",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@cocalc/next",
- "version": "0.39.0",
+ "version": "0.39.1",
"license": "SEE LICENSE IN LICENSE.md",
"workspaces": [
"../assets",
diff --git a/src/packages/next/package.json b/src/packages/next/package.json
index 3710bdbe52..220d0bd991 100644
--- a/src/packages/next/package.json
+++ b/src/packages/next/package.json
@@ -1,6 +1,6 @@
{
"name": "@cocalc/next",
- "version": "0.39.0",
+ "version": "0.39.1",
"description": "CoCalc's nextjs server",
"main": "lib/init.js",
"exports": {
@@ -62,10 +62,10 @@
"@cocalc/assets": "^1.6.0",
"@cocalc/backend": "^1.12.1",
"@cocalc/cdn": "^1.10.0",
- "@cocalc/database": "^0.17.1",
+ "@cocalc/database": "^0.17.2",
"@cocalc/frontend": "^1.41.0",
"@cocalc/server": "^0.20.0",
- "@cocalc/util": "^1.32.1",
+ "@cocalc/util": "^1.32.2",
"@types/express": "^4.17.13",
"@types/react": "^17.0.33",
"antd": "^4.18.2",
diff --git a/src/packages/next/pages/policies/thirdparties.tsx b/src/packages/next/pages/policies/thirdparties.tsx
index e446ebaed4..5925e12db4 100644
--- a/src/packages/next/pages/policies/thirdparties.tsx
+++ b/src/packages/next/pages/policies/thirdparties.tsx
@@ -74,6 +74,24 @@ export default function TermsOfService({ customize }) {
+
+
+ reCAPTCHA
+
+
+
Zendesk
diff --git a/src/packages/util/package-lock.json b/src/packages/util/package-lock.json
index abf66479f5..f7d4f56ff8 100644
--- a/src/packages/util/package-lock.json
+++ b/src/packages/util/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@cocalc/util",
- "version": "1.32.1",
+ "version": "1.32.2",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@cocalc/util",
- "version": "1.32.1",
+ "version": "1.32.2",
"license": "SEE LICENSE.md",
"workspaces": [
"."
diff --git a/src/packages/util/package.json b/src/packages/util/package.json
index f35719f5ed..cbff96159f 100644
--- a/src/packages/util/package.json
+++ b/src/packages/util/package.json
@@ -1,6 +1,6 @@
{
"name": "@cocalc/util",
- "version": "1.32.1",
+ "version": "1.32.2",
"description": "CoCalc code shared between the frontend and the backend",
"exports": {
"./*": "./dist/*.js",
diff --git a/src/packages/util/syncstring_schema.js b/src/packages/util/syncstring_schema.js
index a129f52173..36f8b09461 100644
--- a/src/packages/util/syncstring_schema.js
+++ b/src/packages/util/syncstring_schema.js
@@ -88,6 +88,10 @@ schema.syncstrings = {
desc:
"Shared (by all users) configuration settings for editing this file (e.g., which spellcheck language to use).",
},
+ huge: {
+ type: "boolean",
+ desc: "If true, this syncstring contains too many or too large patches to be processed. Hence if this is set, it won't be processed. TODO: implement a better archiving mechanism and then process such 'huge' syncstrings.",
+ }
},
pg_indexes: ["last_active", "archived"],