Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Introduce mimetype DB update occ command #18742

Merged
merged 5 commits into from
Sep 6, 2015
Merged

Introduce mimetype DB update occ command #18742

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9e9ffb1
Allow access to full alias and mapping arrays
Sep 1, 2015
79fceef
Move maintenance:mimetypesjs to sublocation, cleanup code
Sep 1, 2015
cdf01f0
Split mimetype handling to new class
Sep 3, 2015
19830e6
Introduce mimetype DB update occ command
Sep 1, 2015
c6314fc
Unit tests for mimetype loader
Sep 4, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions apps/files_sharing/lib/cache.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ class Shared_Cache extends Cache {
* @param \OC\Files\Storage\Shared $storage
*/
public function __construct($storage) {
parent::__construct($storage);
$this->storage = $storage;
}

Expand Down Expand Up @@ -94,6 +95,7 @@ public function getNumericStorageId() {
* @return array|false
*/
public function get($file) {
$mimetypeLoader = \OC::$server->getMimeTypeLoader();
if (is_string($file)) {
$cache = $this->getSourceCache($file);
if ($cache) {
Expand Down Expand Up @@ -130,8 +132,8 @@ public function get($file) {
$data['mtime'] = (int)$data['mtime'];
$data['storage_mtime'] = (int)$data['storage_mtime'];
$data['encrypted'] = (bool)$data['encrypted'];
$data['mimetype'] = $this->getMimetype($data['mimetype']);
$data['mimepart'] = $this->getMimetype($data['mimepart']);
$data['mimetype'] = $mimetypeLoader->getMimetypeById($data['mimetype']);
$data['mimepart'] = $mimetypeLoader->getMimetypeById($data['mimepart']);
if ($data['storage_mtime'] === 0) {
$data['storage_mtime'] = $data['mtime'];
}
Expand Down
96 changes: 96 additions & 0 deletions core/command/maintenance/mimetype/updatedb.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
<?php
/**
* @author Robin McCorkell <rmccorkell@owncloud.com>
*
* @copyright Copyright (c) 2015, ownCloud, Inc.
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*/

namespace OC\Core\Command\Maintenance\Mimetype;

use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Input\InputOption;

use OCP\Files\IMimeTypeDetector;
use OCP\Files\IMimeTypeLoader;

class UpdateDB extends Command {

const DEFAULT_MIMETYPE = 'application/octet-stream';

/** @var IMimeTypeDetector */
protected $mimetypeDetector;

/** @var IMimeTypeLoader */
protected $mimetypeLoader;

public function __construct(
IMimeTypeDetector $mimetypeDetector,
IMimeTypeLoader $mimetypeLoader
) {
parent::__construct();
$this->mimetypeDetector = $mimetypeDetector;
$this->mimetypeLoader = $mimetypeLoader;
}

protected function configure() {
$this
->setName('maintenance:mimetype:update-db')
->setDescription('Update database mimetypes and update filecache')
->addOption(
'repair-filecache',
null,
InputOption::VALUE_NONE,
'Repair filecache for all mimetypes, not just new ones'
)
;
}

protected function execute(InputInterface $input, OutputInterface $output) {
$mappings = $this->mimetypeDetector->getAllMappings();

$totalFilecacheUpdates = 0;
$totalNewMimetypes = 0;

foreach ($mappings as $ext => $mimetypes) {
if ($ext[0] === '_') {
// comment
continue;
}
$mimetype = $mimetypes[0];
$existing = $this->mimetypeLoader->exists($mimetype);
// this will add the mimetype if it didn't exist
$mimetypeId = $this->mimetypeLoader->getId($mimetype);

if (!$existing) {
$output->writeln('Added mimetype "'.$mimetype.'" to database');
$totalNewMimetypes++;
}

if (!$existing || $input->getOption('repair-filecache')) {
$touchedFilecacheRows = $this->mimetypeLoader->updateFilecache($ext, $mimetypeId);
if ($touchedFilecacheRows > 0) {
$output->writeln('Updated '.$touchedFilecacheRows.' filecache rows for mimetype "'.$mimetype.'"');
}
$totalFilecacheUpdates += $touchedFilecacheRows;
}
}

$output->writeln('Added '.$totalNewMimetypes.' new mimetypes');
$output->writeln('Updated '.$totalFilecacheUpdates.' filecache rows');
}
}
43 changes: 25 additions & 18 deletions core/command/maintenance/mimetypesjs.php → ...command/maintenance/mimetype/updatejs.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,35 @@
*
*/

namespace OC\Core\Command\Maintenance;
namespace OC\Core\Command\Maintenance\Mimetype;

use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;

class MimeTypesJS extends Command {
use OCP\Files\IMimeTypeDetector;

class UpdateJS extends Command {

/** @var IMimeTypeDetector */
protected $mimetypeDetector;

public function __construct(
IMimeTypeDetector $mimetypeDetector
) {
parent::__construct();
$this->mimetypeDetector = $mimetypeDetector;
}

protected function configure() {
$this
->setName('maintenance:mimetypesjs')
->setName('maintenance:mimetype:update-js')
->setDescription('Update mimetypelist.js');
}

protected function execute(InputInterface $input, OutputInterface $output) {
// Fetch all the aliases
$aliases = json_decode(file_get_contents(\OC::$SERVERROOT . '/config/mimetypealiases.dist.json'), true);

if (file_exists(\OC::$SERVERROOT . '/config/mimetypealiases.json')) {
$custom = get_object_vars(json_decode(file_get_contents(\OC::$SERVERROOT . '/config/mimetypealiases.json')));
$aliases = array_merge($aliases, $custom);
}
$aliases = $this->mimetypeDetector->getAllAliases();

// Remove comments
$keys = array_filter(array_keys($aliases), function($k) {
Expand All @@ -49,23 +57,22 @@ protected function execute(InputInterface $input, OutputInterface $output) {
}

// Fetch all files
$dir = new \DirectoryIterator(dirname(__DIR__) . '/../img/filetypes');
$dir = new \DirectoryIterator(\OC::$SERVERROOT.'/core/img/filetypes');

$files = [];
foreach($dir as $fileInfo) {
if ($fileInfo->isFile()) {
$file = preg_replace('/.[^.]*$/', '', $fileInfo->getFilename());
$files[] = $file;
}
if ($fileInfo->isFile()) {
$file = preg_replace('/.[^.]*$/', '', $fileInfo->getFilename());
$files[] = $file;
}
}

//Remove duplicates
$files = array_values(array_unique($files));


// Fetch all themes!
$themes = [];
$dirs = new \DirectoryIterator(dirname(__DIR__) . '/../../themes/');
$dirs = new \DirectoryIterator(\OC::$SERVERROOT.'/themes/');
foreach($dirs as $dir) {
//Valid theme dir
if ($dir->isFile() || $dir->isDot()) {
Expand All @@ -84,7 +91,7 @@ protected function execute(InputInterface $input, OutputInterface $output) {
$themeIt = new \DirectoryIterator($themeDir);
foreach ($themeIt as $fileInfo) {
if ($fileInfo->isFile()) {
$file = preg_replace('/.[^.]*$/', '', $fileInfo->getFilename());
$file = preg_replace('/.[^.]*$/', '', $fileInfo->getFilename());
$themes[$theme][] = $file;
}
}
Expand All @@ -110,7 +117,7 @@ protected function execute(InputInterface $input, OutputInterface $output) {
';

//Output the JS
file_put_contents(dirname(__DIR__) . '/../js/mimetypelist.js', $js);
file_put_contents(\OC::$SERVERROOT.'/core/js/mimetypelist.js', $js);

$output->writeln('<info>mimetypelist.js is updated');
}
Expand Down
3 changes: 2 additions & 1 deletion core/register_command.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,8 @@
);
$application->add(new OC\Core\Command\Encryption\ShowKeyStorageRoot($util));

$application->add(new OC\Core\Command\Maintenance\MimeTypesJS());
$application->add(new OC\Core\Command\Maintenance\Mimetype\UpdateDB(\OC::$server->getMimeTypeDetector(), \OC::$server->getMimeTypeLoader()));
$application->add(new OC\Core\Command\Maintenance\Mimetype\UpdateJS(\OC::$server->getMimeTypeDetector()));
$application->add(new OC\Core\Command\Maintenance\Mode(\OC::$server->getConfig()));
$application->add(new OC\Core\Command\Maintenance\Repair(new \OC\Repair(\OC\Repair::getRepairSteps()), \OC::$server->getConfig()));
$application->add(new OC\Core\Command\Maintenance\SingleUser(\OC::$server->getConfig()));
Expand Down
96 changes: 16 additions & 80 deletions lib/private/files/cache/cache.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@

namespace OC\Files\Cache;

use \OCP\Files\IMimeTypeLoader;

/**
* Metadata cache for a storage
*
Expand Down Expand Up @@ -66,8 +68,8 @@ class Cache {
*/
protected $storageCache;

protected static $mimetypeIds = array();
protected static $mimetypes = array();
/** @var IMimeTypeLoader */
protected $mimetypeLoader;

/**
* @param \OC\Files\Storage\Storage|string $storage
Expand All @@ -83,6 +85,7 @@ public function __construct($storage) {
}

$this->storageCache = new Storage($storage);
$this->mimetypeLoader = \OC::$server->getMimeTypeLoader();
}

/**
Expand All @@ -94,72 +97,6 @@ public function getNumericStorageId() {
return $this->storageCache->getNumericId();
}

/**
* Get the numeric id for a mimetype
*
* Mimetypes are stored as integers in the cache to prevent duplicated data of the (usually) fairly limited amount of unique mimetypes
* If the supplied mimetype does not yet have a numeric id a new one will be generated
*
* @param string $mime
* @return int
*/
public function getMimetypeId($mime) {
if (empty($mime)) {
// Can not insert empty string into Oracle NOT NULL column.
$mime = 'application/octet-stream';
}
if (empty(self::$mimetypeIds)) {
$this->loadMimetypes();
}

if (!isset(self::$mimetypeIds[$mime])) {
try {
$connection = \OC_DB::getConnection();
$connection->insertIfNotExist('*PREFIX*mimetypes', [
'mimetype' => $mime,
]);
$this->loadMimetypes();
} catch (\Doctrine\DBAL\DBALException $e) {
\OCP\Util::writeLog('core', 'Exception during mimetype insertion: ' . $e->getmessage(), \OCP\Util::DEBUG);
return -1;
}
}

return self::$mimetypeIds[$mime];
}


/**
* Get the mimetype (as string) from a mimetype id
*
* @param int $id
* @return string | null the mimetype for the id or null if the id is not known
*/
public function getMimetype($id) {
if (empty(self::$mimetypes)) {
$this->loadMimetypes();
}

return isset(self::$mimetypes[$id]) ? self::$mimetypes[$id] : null;
}

/**
* Load all known mimetypes and mimetype ids from the database
*
* @throws \OC\DatabaseException
*/
public function loadMimetypes() {
self::$mimetypeIds = self::$mimetypes = array();

$result = \OC_DB::executeAudited('SELECT `id`, `mimetype` FROM `*PREFIX*mimetypes`', array());
if ($result) {
while ($row = $result->fetchRow()) {
self::$mimetypeIds[$row['mimetype']] = $row['id'];
self::$mimetypes[$row['id']] = $row['mimetype'];
}
}
}

/**
* get the stored metadata of a file or folder
*
Expand Down Expand Up @@ -222,8 +159,8 @@ public function get($file) {
$data['storage_mtime'] = (int)$data['storage_mtime'];
$data['encrypted'] = (bool)$data['encrypted'];
$data['storage'] = $this->storageId;
$data['mimetype'] = $this->getMimetype($data['mimetype']);
$data['mimepart'] = $this->getMimetype($data['mimepart']);
$data['mimetype'] = $this->mimetypeLoader->getMimetypeById($data['mimetype']);
$data['mimepart'] = $this->mimetypeLoader->getMimetypeById($data['mimepart']);
if ($data['storage_mtime'] == 0) {
$data['storage_mtime'] = $data['mtime'];
}
Expand Down Expand Up @@ -258,8 +195,8 @@ public function getFolderContentsById($fileId) {
$result = \OC_DB::executeAudited($sql, array($fileId));
$files = $result->fetchAll();
foreach ($files as &$file) {
$file['mimetype'] = $this->getMimetype($file['mimetype']);
$file['mimepart'] = $this->getMimetype($file['mimepart']);
$file['mimetype'] = $this->mimetypeLoader->getMimetypeById($file['mimetype']);
$file['mimepart'] = $this->mimetypeLoader->getMimetypeById($file['mimepart']);
if ($file['storage_mtime'] == 0) {
$file['storage_mtime'] = $file['mtime'];
}
Expand Down Expand Up @@ -385,9 +322,9 @@ protected function buildParts(array $data) {
$params[] = md5($value);
$queryParts[] = '`path_hash`';
} elseif ($name === 'mimetype') {
$params[] = $this->getMimetypeId(substr($value, 0, strpos($value, '/')));
$params[] = $this->mimetypeLoader->getId(substr($value, 0, strpos($value, '/')));
$queryParts[] = '`mimepart`';
$value = $this->getMimetypeId($value);
$value = $this->mimetypeLoader->getId($value);
} elseif ($name === 'storage_mtime') {
if (!isset($data['mtime'])) {
$params[] = $value;
Expand Down Expand Up @@ -613,7 +550,6 @@ public function getStatus($file) {
* @return array an array of cache entries where the name matches the search pattern
*/
public function search($pattern) {

// normalize pattern
$pattern = $this->normalize($pattern);

Expand All @@ -630,8 +566,8 @@ public function search($pattern) {

$files = array();
while ($row = $result->fetchRow()) {
$row['mimetype'] = $this->getMimetype($row['mimetype']);
$row['mimepart'] = $this->getMimetype($row['mimepart']);
$row['mimetype'] = $this->mimetypeLoader->getMimetypeById($row['mimetype']);
$row['mimepart'] = $this->mimetypeLoader->getMimetypeById($row['mimepart']);
$files[] = $row;
}
return $files;
Expand All @@ -652,12 +588,12 @@ public function searchByMime($mimetype) {
}
$sql = 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag`, `permissions`
FROM `*PREFIX*filecache` WHERE ' . $where . ' AND `storage` = ?';
$mimetype = $this->getMimetypeId($mimetype);
$mimetype = $this->mimetypeLoader->getId($mimetype);
$result = \OC_DB::executeAudited($sql, array($mimetype, $this->getNumericStorageId()));
$files = array();
while ($row = $result->fetchRow()) {
$row['mimetype'] = $this->getMimetype($row['mimetype']);
$row['mimepart'] = $this->getMimetype($row['mimepart']);
$row['mimetype'] = $this->mimetypeLoader->getMimetypeById($row['mimetype']);
$row['mimepart'] = $this->mimetypeLoader->getMimetypeById($row['mimepart']);
$files[] = $row;
}
return $files;
Expand Down
Loading