Skip to content

Commit b4d7498

Browse files
sorbaughsorbaugh
authored
Merge pull request #46639 from nextcloud/autosharding
Transparent* database sharding
2 parents c30c9d4 + 2574cbf commit b4d7498

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+3570
-167
lines changed

‎.github/workflows/phpunit-mysql-sharding.yml

+184
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
# This workflow is provided via the organization template repository
2+
#
3+
# https://github.com/nextcloud/.github
4+
# https://docs.github.com/en/actions/learn-github-actions/sharing-workflows-with-your-organization
5+
#
6+
# SPDX-FileCopyrightText: 2022-2024 Nextcloud GmbH and Nextcloud contributors
7+
# SPDX-License-Identifier: MIT
8+
9+
name: PHPUnit sharding
10+
11+
on:
12+
pull_request:
13+
schedule:
14+
- cron: "5 2 * * *"
15+
16+
permissions:
17+
contents: read
18+
19+
concurrency:
20+
group: phpunit-mysql-sharding-${{ github.head_ref || github.run_id }}
21+
cancel-in-progress: true
22+
23+
jobs:
24+
changes:
25+
runs-on: ubuntu-latest-low
26+
27+
outputs:
28+
src: ${{ steps.changes.outputs.src }}
29+
30+
steps:
31+
- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
32+
id: changes
33+
continue-on-error: true
34+
with:
35+
filters: |
36+
src:
37+
- '.github/workflows/**'
38+
- '3rdparty/**'
39+
- '**/appinfo/**'
40+
- '**/lib/**'
41+
- '**/templates/**'
42+
- '**/tests/**'
43+
- 'vendor/**'
44+
- 'vendor-bin/**'
45+
- '.php-cs-fixer.dist.php'
46+
- 'composer.json'
47+
- 'composer.lock'
48+
- '**.php'
49+
50+
phpunit-mysql:
51+
runs-on: ubuntu-latest
52+
53+
needs: changes
54+
if: needs.changes.outputs.src != 'false'
55+
56+
strategy:
57+
matrix:
58+
php-versions: ['8.1']
59+
mysql-versions: ['8.4']
60+
61+
name: Sharding - MySQL ${{ matrix.mysql-versions }} (PHP ${{ matrix.php-versions }}) - database tests
62+
63+
services:
64+
cache:
65+
image: ghcr.io/nextcloud/continuous-integration-redis:latest
66+
ports:
67+
- 6379:6379/tcp
68+
options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3
69+
70+
mysql:
71+
image: ghcr.io/nextcloud/continuous-integration-mysql-${{ matrix.mysql-versions }}:latest
72+
ports:
73+
- 4444:3306/tcp
74+
env:
75+
MYSQL_ROOT_PASSWORD: rootpassword
76+
MYSQL_USER: oc_autotest
77+
MYSQL_PASSWORD: nextcloud
78+
MYSQL_DATABASE: oc_autotest
79+
options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 10
80+
shard1:
81+
image: ghcr.io/nextcloud/continuous-integration-mysql-${{ matrix.mysql-versions }}:latest
82+
ports:
83+
- 5001:3306/tcp
84+
env:
85+
MYSQL_ROOT_PASSWORD: rootpassword
86+
MYSQL_USER: oc_autotest
87+
MYSQL_PASSWORD: nextcloud
88+
MYSQL_DATABASE: nextcloud
89+
options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 10
90+
shard2:
91+
image: ghcr.io/nextcloud/continuous-integration-mysql-${{ matrix.mysql-versions }}:latest
92+
ports:
93+
- 5002:3306/tcp
94+
env:
95+
MYSQL_ROOT_PASSWORD: rootpassword
96+
MYSQL_USER: oc_autotest
97+
MYSQL_PASSWORD: nextcloud
98+
MYSQL_DATABASE: nextcloud
99+
options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 10
100+
shard3:
101+
image: ghcr.io/nextcloud/continuous-integration-mysql-${{ matrix.mysql-versions }}:latest
102+
ports:
103+
- 5003:3306/tcp
104+
env:
105+
MYSQL_ROOT_PASSWORD: rootpassword
106+
MYSQL_USER: oc_autotest
107+
MYSQL_PASSWORD: nextcloud
108+
MYSQL_DATABASE: nextcloud
109+
options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 10
110+
shard4:
111+
image: ghcr.io/nextcloud/continuous-integration-mysql-${{ matrix.mysql-versions }}:latest
112+
ports:
113+
- 5004:3306/tcp
114+
env:
115+
MYSQL_ROOT_PASSWORD: rootpassword
116+
MYSQL_USER: oc_autotest
117+
MYSQL_PASSWORD: nextcloud
118+
MYSQL_DATABASE: nextcloud
119+
options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 10
120+
121+
steps:
122+
- name: Checkout server
123+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
124+
with:
125+
submodules: true
126+
127+
- name: Set up php ${{ matrix.php-versions }}
128+
uses: shivammathur/setup-php@2e947f1f6932d141d076ca441d0e1e881775e95b #v2.31.0
129+
with:
130+
php-version: ${{ matrix.php-versions }}
131+
# https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation
132+
extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, mysql, pdo_mysql
133+
coverage: ${{ matrix.coverage && 'xdebug' || 'none' }}
134+
ini-file: development
135+
env:
136+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
137+
138+
- name: Set up dependencies
139+
run: composer i
140+
141+
- name: Enable ONLY_FULL_GROUP_BY MySQL option
142+
run: |
143+
echo "SET GLOBAL sql_mode=(SELECT CONCAT(@@sql_mode,',ONLY_FULL_GROUP_BY'));" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword
144+
echo "SELECT @@sql_mode;" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword
145+
146+
- name: Set up Nextcloud
147+
env:
148+
DB_PORT: 4444
149+
SHARDING: 1
150+
run: |
151+
mkdir data
152+
cp tests/redis.config.php config/
153+
cp tests/preseed-config.php config/config.php
154+
./occ maintenance:install --verbose --database=mysql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin
155+
php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0
156+
157+
- name: PHPUnit
158+
run: composer run test:db ${{ matrix.coverage && ' -- --coverage-clover ./clover.db.xml' || '' }}
159+
160+
- name: Upload db code coverage
161+
if: ${{ !cancelled() && matrix.coverage }}
162+
uses: codecov/codecov-action@v4.1.1
163+
with:
164+
files: ./clover.db.xml
165+
flags: phpunit-mysql
166+
167+
- name: Print logs
168+
if: always()
169+
run: |
170+
cat data/nextcloud.log
171+
172+
summary:
173+
permissions:
174+
contents: none
175+
runs-on: ubuntu-latest-low
176+
needs: [changes, phpunit-mysql]
177+
178+
if: always()
179+
180+
name: phpunit-mysql-summary
181+
182+
steps:
183+
- name: Summary status
184+
run: if ${{ needs.changes.outputs.src != 'false' && needs.phpunit-mysql.result != 'success' }}; then exit 1; fi

‎apps/dav/lib/DAV/CustomPropertiesBackend.php

+9-9
Original file line numberDiff line numberDiff line change
@@ -364,16 +364,16 @@ private function getPublishedProperties(string $path, array $requestedProperties
364364
private function cacheDirectory(string $path, Directory $node): void {
365365
$prefix = ltrim($path . '/', '/');
366366
$query = $this->connection->getQueryBuilder();
367-
$query->select('name', 'propertypath', 'propertyname', 'propertyvalue', 'valuetype')
367+
$query->select('name', 'p.propertypath', 'p.propertyname', 'p.propertyvalue', 'p.valuetype')
368368
->from('filecache', 'f')
369-
->leftJoin('f', 'properties', 'p', $query->expr()->andX(
370-
$query->expr()->eq('propertypath', $query->func()->concat(
371-
$query->createNamedParameter($prefix),
372-
'name'
373-
)),
374-
$query->expr()->eq('userid', $query->createNamedParameter($this->user->getUID()))
375-
))
376-
->where($query->expr()->eq('parent', $query->createNamedParameter($node->getInternalFileId(), IQueryBuilder::PARAM_INT)));
369+
->hintShardKey('storage', $node->getNode()->getMountPoint()->getNumericStorageId())
370+
->leftJoin('f', 'properties', 'p', $query->expr()->eq('p.propertypath', $query->func()->concat(
371+
$query->createNamedParameter($prefix),
372+
'f.name'
373+
)),
374+
)
375+
->where($query->expr()->eq('parent', $query->createNamedParameter($node->getInternalFileId(), IQueryBuilder::PARAM_INT)))
376+
->andWhere($query->expr()->eq('p.userid', $query->createNamedParameter($this->user->getUID())));
377377
$result = $query->executeQuery();
378378

379379
$propsByPath = [];

‎apps/files/lib/BackgroundJob/DeleteOrphanedItems.php

+69-17
Original file line numberDiff line numberDiff line change
@@ -52,34 +52,86 @@ public function run($argument) {
5252
* @param string $typeCol
5353
* @return int Number of deleted entries
5454
*/
55-
protected function cleanUp($table, $idCol, $typeCol) {
55+
protected function cleanUp(string $table, string $idCol, string $typeCol): int {
5656
$deletedEntries = 0;
5757

58-
$query = $this->connection->getQueryBuilder();
59-
$query->select('t1.' . $idCol)
60-
->from($table, 't1')
61-
->where($query->expr()->eq($typeCol, $query->expr()->literal('files')))
62-
->andWhere($query->expr()->isNull('t2.fileid'))
63-
->leftJoin('t1', 'filecache', 't2', $query->expr()->eq($query->expr()->castColumn('t1.' . $idCol, IQueryBuilder::PARAM_INT), 't2.fileid'))
64-
->groupBy('t1.' . $idCol)
65-
->setMaxResults(self::CHUNK_SIZE);
66-
6758
$deleteQuery = $this->connection->getQueryBuilder();
6859
$deleteQuery->delete($table)
69-
->where($deleteQuery->expr()->in($idCol, $deleteQuery->createParameter('objectid')));
60+
->where($deleteQuery->expr()->eq($idCol, $deleteQuery->createParameter('objectid')));
61+
62+
if ($this->connection->getShardDefinition('filecache')) {
63+
$sourceIdChunks = $this->getItemIds($table, $idCol, $typeCol, 1000);
64+
foreach ($sourceIdChunks as $sourceIdChunk) {
65+
$deletedSources = $this->findMissingSources($sourceIdChunk);
66+
$deleteQuery->setParameter('objectid', $deletedSources, IQueryBuilder::PARAM_INT_ARRAY);
67+
$deletedEntries += $deleteQuery->executeStatement();
68+
}
69+
} else {
70+
$query = $this->connection->getQueryBuilder();
71+
$query->select('t1.' . $idCol)
72+
->from($table, 't1')
73+
->where($query->expr()->eq($typeCol, $query->expr()->literal('files')))
74+
->leftJoin('t1', 'filecache', 't2', $query->expr()->eq($query->expr()->castColumn('t1.' . $idCol, IQueryBuilder::PARAM_INT), 't2.fileid'))
75+
->andWhere($query->expr()->isNull('t2.fileid'))
76+
->groupBy('t1.' . $idCol)
77+
->setMaxResults(self::CHUNK_SIZE);
78+
79+
$deleteQuery = $this->connection->getQueryBuilder();
80+
$deleteQuery->delete($table)
81+
->where($deleteQuery->expr()->in($idCol, $deleteQuery->createParameter('objectid')));
7082

71-
$deletedInLastChunk = self::CHUNK_SIZE;
72-
while ($deletedInLastChunk === self::CHUNK_SIZE) {
73-
$chunk = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
74-
$deletedInLastChunk = count($chunk);
83+
$deletedInLastChunk = self::CHUNK_SIZE;
84+
while ($deletedInLastChunk === self::CHUNK_SIZE) {
85+
$chunk = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
86+
$deletedInLastChunk = count($chunk);
7587

76-
$deleteQuery->setParameter('objectid', $chunk, IQueryBuilder::PARAM_INT_ARRAY);
77-
$deletedEntries += $deleteQuery->executeStatement();
88+
$deleteQuery->setParameter('objectid', $chunk, IQueryBuilder::PARAM_INT_ARRAY);
89+
$deletedEntries += $deleteQuery->executeStatement();
90+
}
7891
}
7992

8093
return $deletedEntries;
8194
}
8295

96+
/**
97+
* @param string $table
98+
* @param string $idCol
99+
* @param string $typeCol
100+
* @param int $chunkSize
101+
* @return \Iterator<int[]>
102+
* @throws \OCP\DB\Exception
103+
*/
104+
private function getItemIds(string $table, string $idCol, string $typeCol, int $chunkSize): \Iterator {
105+
$query = $this->connection->getQueryBuilder();
106+
$query->select($idCol)
107+
->from($table)
108+
->where($query->expr()->eq($typeCol, $query->expr()->literal('files')))
109+
->groupBy($idCol)
110+
->andWhere($query->expr()->gt($idCol, $query->createParameter('min_id')))
111+
->setMaxResults($chunkSize);
112+
113+
$minId = 0;
114+
while (true) {
115+
$query->setParameter('min_id', $minId);
116+
$rows = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
117+
if (count($rows) > 0) {
118+
$minId = $rows[count($rows) - 1];
119+
yield $rows;
120+
} else {
121+
break;
122+
}
123+
}
124+
}
125+
126+
private function findMissingSources(array $ids): array {
127+
$qb = $this->connection->getQueryBuilder();
128+
$qb->select('fileid')
129+
->from('filecache')
130+
->where($qb->expr()->in('fileid', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY)));
131+
$found = $qb->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
132+
return array_diff($ids, $found);
133+
}
134+
83135
/**
84136
* Deleting orphaned system tag mappings
85137
*

0 commit comments

Comments
 (0)