ffsync: Update to syncstorage-rs

cross/syncstorage-rs/Makefile

@@ -0,0 +1,27 @@
+PKG_NAME = syncstorage-rs
+PKG_VERS = 0.14.1
+PKG_EXT = tar.gz
+PKG_DIST_NAME = $(PKG_NAME)-$(PKG_VERS).$(PKG_EXT)
+PKG_DIST_SITE = https://github.com/mozilla-services/$(PKG_NAME)/archive/${PKG_VERS}
+PKG_DIR = $(PKG_NAME)-$(PKG_VERS)
+
+DEPENDS =
+
+# powerpc archs (except qoriq) are not supported
+UNSUPPORTED_ARCHS += $(OLD_PPC_ARCHS)
+
+HOMEPAGE = https://github.com/mozilla-services/syncstorage-rs/
+COMMENT  = Mozilla Sync Storage built with Rust.
+LICENSE  = https://github.com/mozilla-services/syncstorage-rs/blob/master/LICENSE
+
+# Mandatory for rustc wheel building
+ENV += PYO3_CROSS_LIB_DIR=$(STAGING_INSTALL_PREFIX)/lib/
+ENV += PYO3_CROSS_INCLUDE_DIR=$(STAGING_INSTALL_PREFIX)/include/
+
+# arguments to install the main server program
+RUST_SRC_DIR = $(WORK_DIR)/$(PKG_DIR)/syncserver
+CARGO_BUILD_ARGS += --no-default-features
+CARGO_BUILD_ARGS += --features=syncstorage-db/mysql
+CARGO_BUILD_ARGS += --locked
+
+include ../../mk/spksrc.cross-rust.mk

cross/syncstorage-rs/digests

@@ -0,0 +1,3 @@
+syncstorage-rs-0.14.1.tar.gz SHA1 2b4fd69655caf646bed046fcc85db4ef844f0746
+syncstorage-rs-0.14.1.tar.gz SHA256 224d8c4af114b0af4eebd90decef951a08dde1204c13caada416347aa5ebe35c
+syncstorage-rs-0.14.1.tar.gz MD5 de9034fc2fcc5bcbccf9df5c0e86f405

mk/spksrc.cross-rust.mk

@@ -53,6 +53,11 @@ endif
 CARGO_INSTALL_ARGS += --path $(RUST_SRC_DIR)
 CARGO_INSTALL_ARGS += --root $(STAGING_INSTALL_PREFIX)
 
+# Append additional install options if present
+ifneq ($(strip $(CARGO_BUILD_ARGS)),)
+CARGO_INSTALL_ARGS += $(CARGO_BUILD_ARGS)
+endif
+
 # Default build with rust and install with cargo
 rust_install_target:
 	@echo "  ==> Cargo install rust package $(PKG_NAME) ($(shell rustc --version); $(RUST_TOOLCHAIN))"

spk/ffsync/Makefile

@@ -1,47 +1,55 @@
 SPK_NAME = ffsync
-SPK_VERS = 1.5.2
-SPK_REV = 3
+SPK_VERS = 0.14.1
+SPK_REV = 4
 SPK_ICON = src/ffsync.png
-BETA = 1
 
-BUILD_DEPENDS = cross/python2 cross/setuptools_py2 cross/pip_py2 cross/wheel
-BUILD_DEPENDS += cross/gevent cross/greenlet cross/cffi
+BUILD_DEPENDS = cross/curl cross/mysql-connector-c
+DEPENDS     = cross/syncstorage-rs
 
-DEPENDS     = cross/busybox
-WHEELS      = src/requirements.txt
+PYTHON_PACKAGE = python311
+WHEELS      = src/requirements-crossenv.txt src/requirements-pure.txt
+
+REQUIRED_MIN_DSM = 6.0
+SPK_DEPENDS = MariaDB10:$(PYTHON_PACKAGE)
 
 MAINTAINER   = SynoCommunity
 DESCRIPTION  = Firefox Sync Server 1.5, used for Firefox 29 and later. You can use Firefox Sync Server to synchronize your bookmarks, passwords, settings, history, add-ons and tabs with Firefox on other computers. The service runs on port 8132.
 DISPLAY_NAME = Firefox Sync Server 1.5
-CHANGELOG = "1. Fix Python requirement<br>2. Fix startup issue when pidfile is not yet written"
+CHANGELOG = "1. Upgrade to Mozilla Sync Storage built with Rust."
 
-HOMEPAGE     = https://docs.services.mozilla.com/howtos/run-sync-1.5.html
-LICENSE      = MPL/GPL/LGPL
+HOMEPAGE     = https://mozilla-services.readthedocs.io/en/latest/howtos/run-sync-1.5.html
+LICENSE      = MPL-2.0
 HELPURL      = https://github.com/SynoCommunity/spksrc/wiki/Firefox-Sync-Server-1.5
 
-INSTALLER_SCRIPT = src/installer.sh
-SSS_SCRIPT       = src/dsm-control.sh
-FWPORTS          = src/${SPK_NAME}.sc
+WIZARDS_DIR = src/wizard/
+
+STARTABLE = yes
+SERVICE_USER = auto
+SERVICE_SETUP = src/service-setup.sh
+SERVICE_PORT = 8132
+SERVICE_PORT_TITLE = $(DISPLAY_NAME)
+
+# Admin link for in DSM UI
+ADMIN_PORT = $(SERVICE_PORT)
+
 CONF_DIR         = src/conf/
 
-INSTALL_DEP_SERVICES = mysql
-START_DEP_SERVICES = mysql
+# [mysqlclient]
+DEPENDS += cross/mysql-connector-c cross/mariadb-connector-c
+ENV += MYSQLCLIENT_CFLAGS="$(CFLAGS) -I$(STAGING_INSTALL_PREFIX)/include/mysql -I$(STAGING_INSTALL_PREFIX)/include/mariadb -I$(STAGING_INSTALL_PREFIX)/$(PYTHON_INC_DIR)"
+ENV += MYSQLCLIENT_LDFLAGS="$(LDFLAGS)"
 
-INSTALL_PREFIX = /usr/local/$(SPK_NAME)
+# [bcrypt] and [cryptography]
+# Mandatory for rustc wheel building
+ENV += PYO3_CROSS_LIB_DIR=$(STAGING_INSTALL_PREFIX)/lib/
+ENV += PYO3_CROSS_INCLUDE_DIR=$(STAGING_INSTALL_PREFIX)/include/
+ENV += CRYPTOGRAPHY_DONT_BUILD_RUST=1
 
 POST_STRIP_TARGET = ffsync_extra_install
 
-BUSYBOX_CONFIG  = usermng
-ENV += BUSYBOX_CONFIG="$(BUSYBOX_CONFIG)"
-
-include ../../mk/spksrc.spk.mk
+include ../../mk/spksrc.python.mk
 
 .PHONY: ffsync_extra_install
 ffsync_extra_install:
 	install -m 755 -d $(STAGING_DIR)/var
 	install -m 644 src/ffsync.ini $(STAGING_DIR)/var/
-	sed -i -e "s|https://github.com/mozilla-services/mozservices/archive/e00e1b68130423ad98d0f6185655bde650443da8.zip|mozsvc==0.8|g" \
-	       -e "s|https://github.com/mozilla-services/tokenserver/archive/d7e513e8a4f5c588b70d685a8df1d2e508c341c0.zip|tokenserver==1.2.7|g" \
-	       -e "s|https://github.com/mozilla-services/server-syncstorage/archive/1.5.5.zip|SyncStorage==1.5.5|g" \
-	       -e "s|https://github.com/mozilla-services/syncserver/archive/1.5.2.tar.gz|syncserver==1.5.2|g" \
-                $(STAGING_DIR)/share/wheelhouse/requirements.txt

spk/ffsync/src/requirements-all.txt

@@ -0,0 +1,21 @@
+# From requirements.txt
+
+# We pin cryptography to 3.4.8 due to a build error that occurs with the
+# current version (36.0.0 at the time of this commit). In short, building
+# the cryptography package with Rust results in a segfault, so we need to
+# set the CRYPTOGRAPHY_DONT_BUILD_RUST env var to force the package to build
+# with C instead. This env var is only present in cryptography<3.5.
+cryptography==3.4.8
+pyfxa==0.7.7
+tokenlib==2.0.0
+
+# From tools/tokenserver/requirements.txt
+
+boto==2.49.0
+hawkauthlib==2.0.0
+mysqlclient==2.1.1
+pyramid==2.0.1
+sqlalchemy==1.4.46
+testfixtures
+tokenlib==2.0.0
+PyBrowserID==0.14.0

spk/ffsync/src/requirements-crossenv.txt

@@ -0,0 +1,24 @@
+# From requirements.txt
+
+# We pin cryptography to 3.4.8 due to a build error that occurs with the
+# current version (36.0.0 at the time of this commit). In short, building
+# the cryptography package with Rust results in a segfault, so we need to
+# set the CRYPTOGRAPHY_DONT_BUILD_RUST env var to force the package to build
+# with C instead. This env var is only present in cryptography<3.5.
+
+# [bcrypt] & [cryptography]
+# Require environment variables
+#  PYO3_CROSS_LIB_DIR=$(STAGING_INSTALL_PREFIX)/lib/
+#  PYO3_CROSS_INCLUDE_DIR=$(STAGING_INSTALL_PREFIX)/include/
+cryptography==3.4.8
+
+# From tools/tokenserver/requirements.txt
+
+# [mysqlclient]
+# Depends: mysql-connector-c, mariadb-connector-c
+# Require environment variables
+#  MYSQLCLIENT_CFLAGS
+#  MYSQLCLIENT_LDFLAGS
+mysqlclient==2.1.1
+
+sqlalchemy==1.4.46

spk/ffsync/src/requirements-pure.txt

@@ -0,0 +1,14 @@
+# From requirements.txt
+
+pyfxa==0.7.7
+tokenlib==2.0.0
+
+# From tools/tokenserver/requirements.txt
+
+boto==2.49.0
+hawkauthlib==2.0.0
+pyramid==2.0.1
+sqlalchemy==1.4.46
+testfixtures
+tokenlib==2.0.0
+PyBrowserID==0.14.0

spk/ffsync/src/service-setup.sh

@@ -0,0 +1,103 @@
+PYTHON_DIR="/var/packages/python311/target/bin"
+PATH="${SYNOPKG_PKGDEST}/env/bin:${SYNOPKG_PKGDEST}/bin:${PYTHON_DIR}:${PATH}"
+MARIADB_10_INSTALL_DIRECTORY="/var/packages/MariaDB10"
+MARIADB_10_BIN_DIRECTORY="${MARIADB_10_INSTALL_DIRECTORY}/target/usr/local/mariadb10/bin"
+MYSQL="${MARIADB_10_BIN_DIRECTORY}/mysql"
+
+PYTHON="${SYNOPKG_PKGDEST}/env/bin/python3"
+SYNCSERVER="${HOME}/bin/syncserver"
+HOME="${SYNOPKG_PKGDEST}/share/${SPK_NAME}"
+CFG_FILE="${HOME}/config/local.toml"
+
+SERVICE_COMMAND="${SYNCSERVER} --config=${CFG_FILE}"
+
+service_preinst ()
+{
+    # Check MySQL database
+    if [ "${SYNOPKG_PKG_STATUS}" = "INSTALL" ]; then
+        if [ -n "${wizard_mysql_password_root}" ]; then
+            if ! ${MYSQL} -u root -p"${wizard_mysql_password_root}" -e quit > /dev/null 2>&1; then
+                echo "Incorrect MySQL root password"
+                exit 1
+            fi
+            if ${MYSQL} -u root -p"${wizard_mysql_password_root}" mysql -e "SELECT User FROM user" | grep ^${SPK_NAME}$ > /dev/null 2>&1; then
+                echo "MySQL user ${SPK_NAME} already exists"
+                exit 1
+            fi
+            if ${MYSQL} -u root -p"${wizard_mysql_password_root}" -e "SHOW DATABASES" | grep -E '^(syncstorage_rs|tokenserver_rs)$' > /dev/null 2>&1; then
+                echo "MySQL database(s) for ${SPK_NAME} already exist(s)"
+                exit 1
+            fi
+        fi   
+    fi
+}
+
+service_postinst ()
+{
+    # Create a Python virtualenv
+    install_python_virtualenv
+
+    # Install wheels
+    install_python_wheels
+
+    if [ "${SYNOPKG_PKG_STATUS}" = "INSTALL" ]; then
+        # login as root sql user using whatever creds you set up for that
+        # this sets up a user for sync storage and sets up the databases
+        ${MYSQL} -u root -p ${wizard_mysql_password_root} <<EOF
+CREATE USER "${SPK_NAME}"@"localhost" IDENTIFIED BY "${wizard_password_ffsync}";
+CREATE DATABASE syncstorage_rs;
+CREATE DATABASE tokenserver_rs;
+GRANT ALL PRIVILEGES on syncstorage_rs.* to ${SPK_NAME}@localhost;
+GRANT ALL PRIVILEGES on tokenserver_rs.* to ${SPK_NAME}@localhost;
+EOF
+
+        # Run initial database migrations
+        # syncstorage db initialization
+        $HOME/.cargo/bin/diesel --database-url "mysql://${SPK_NAME}:${wizard_password_ffsync}@localhost/syncstorage_rs" \
+            migration --migration-dir syncstorage-mysql/migrations run
+        # tokenserver db initialization
+        $HOME/.cargo/bin/diesel --database-url "mysql://${SPK_NAME}:${wizard_password_ffsync}@localhost/tokenserver_rs" \
+            migration --migration-dir tokenserver-db/migrations run
+
+        # Add sync endpoint to database
+        ${MYSQL} -u ${SPK_NAME} -p "$(wizard_password_ffsync)" <<EOF
+USE tokenserver_rs
+INSERT INTO services (id, service, pattern) VALUES
+    (1, "sync-1.5", "{node}/1.5/{uid}");
+EOF
+
+        # Add syncserver node
+        # the 10 is the user capacity.
+        SYNC_TOKENSERVER__DATABASE_URL="mysql://${SPK_NAME}:${wizard_password_ffsync}@localhost/tokenserver_rs" \
+            python3 tools/tokenserver/add_node.py \
+            ${wizard_ffsync_public_url} 10
+
+        # Setup syncserver config file
+        MASTER_SECRET="$(cat /dev/urandom | base32 | head -c64)"
+        METRICS_HASH_SECRET="$(cat /dev/urandom | base32 | head -c64)"
+        cat > ${CFG_FILE} <<EOF
+master_secret = "${MASTER_SECRET}"
+
+# removing this line will default to moz_json formatted logs
+human_logs = 1
+
+host = "localhost" # default
+port = ${SERVICE_PORT} # SPK default
+
+syncstorage.database_url = "mysql://${SPK_NAME}:${wizard_password_ffsync}@localhost/syncstorage_rs"
+syncstorage.enable_quota = 0
+syncstorage.enabled = true
+syncstorage.limits.max_total_records = 1666 # See issues #298/#333
+
+# token
+tokenserver.database_url = "mysql://${SPK_NAME}:${wizard_password_ffsync}@localhost/tokenserver_rs"
+tokenserver.enabled = true 
+tokenserver.fxa_email_domain = "api.accounts.firefox.com"
+tokenserver.fxa_metrics_hash_secret = "${METRICS_HASH_SECRET}"
+tokenserver.fxa_oauth_server_url = "https://oauth.accounts.firefox.com"
+tokenserver.fxa_browserid_audience = "https://token.services.mozilla.com"
+tokenserver.fxa_browserid_issuer = "https://api.accounts.firefox.com"
+tokenserver.fxa_browserid_server_url = "https://verifier.accounts.firefox.com/v2"
+EOF
+    fi
+}