Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[com_fields] Fetch all custom field values in one query #14558

Merged
merged 12 commits into from
Mar 27, 2017
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,9 @@ CREATE TABLE IF NOT EXISTS `#__fields_groups` (

CREATE TABLE IF NOT EXISTS `#__fields_values` (
`field_id` int(10) unsigned NOT NULL,
`context` varchar(255) NOT NULL,
`item_id` varchar(255) NOT NULL COMMENT 'Allow references to items which have strings as ids, eg. none db systems.',
`value` text NOT NULL DEFAULT '',
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are not changed it now, but database field with type TEXT NOT NULL can not have default value on mysql.
https://dev.mysql.com/doc/refman/5.7/en/blob.html

There is a lot of such mistakes in joomla but please do not repeat it.
I see 2 solutions. Allow to NULL or always set default value in php code.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That sort of stuff should be done in a separate pr, basically on thing in one pr, otherwise it gets harder to review.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I want only mention about it.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No problem.

KEY (`field_id`),
KEY (`context`),
KEY (`item_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_unicode_ci;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,11 +79,9 @@ CREATE INDEX "#__fields_idx_language" ON "#__fields_groups" ("language");
CREATE TABLE "#__fields_values" (
"field_id" bigint DEFAULT 0 NOT NULL,
"item_id" varchar(255) DEFAULT '' NOT NULL,
"context" varchar(255) DEFAULT '' NOT NULL,
"value" text DEFAULT '' NOT NULL
"value" text DEFAULT '' NOT NULL
);
CREATE INDEX "#__fields_values_idx_field_id" ON "#__fields_values" ("field_id");
CREATE INDEX "#__fields_values_idx_context" ON "#__fields_values" ("context");
CREATE INDEX "#__fields_values_idx_item_id" ON "#__fields_values" ("item_id");

INSERT INTO "#__extensions" ("extension_id", "name", "type", "element", "folder", "client_id", "enabled", "access", "protected", "manifest_cache", "params", "custom_data", "system_data", "checked_out", "checked_out_time", "ordering", "state") VALUES
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,6 @@ SET QUOTED_IDENTIFIER ON;

CREATE TABLE [#__fields_values] (
[field_id] [bigint] NOT NULL DEFAULT 1,
[context] [nvarchar](255) NOT NULL DEFAULT '',
[item_id] [nvarchar](255) NOT NULL DEFAULT '',
[value] [nvarchar](max) NOT NULL DEFAULT '',
) ON [PRIMARY];
Expand All @@ -130,10 +129,6 @@ CREATE NONCLUSTERED INDEX [idx_field_id] ON [#__fields_values](
[field_id] ASC)
WITH (STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF);

CREATE NONCLUSTERED INDEX [idx_context] ON [#__fields_values](
[context] ASC)
WITH (STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF);

CREATE NONCLUSTERED INDEX [idx_item_id] ON [#__fields_values](
[item_id] ASC)
WITH (STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF);
Expand Down
20 changes: 10 additions & 10 deletions administrator/components/com_fields/helpers/fields.php
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,10 @@ public static function getFields($context, $item = null, $prepareValue = false,
self::$fieldCache = JModelLegacy::getInstance('Field', 'FieldsModel', array('ignore_request' => true));
}

$fieldIds = array_map(function($f) { return $f->id; }, $fields);

$fieldValues = self::$fieldCache->getFieldValues($fieldIds, $item->id);

$new = array();

foreach ($fields as $key => $original)
Expand All @@ -163,12 +167,12 @@ public static function getFields($context, $item = null, $prepareValue = false,
{
$field->value = $valuesToOverride[$field->id];
}
else
elseif (key_exists($field->id, $fieldValues))
{
$field->value = self::$fieldCache->getFieldValue($field->id, $field->context, $item->id);
$field->value = $fieldValues[$field->id];
}

if ($field->value === '' || $field->value === null)
if (!isset($field->value) || $field->value === '')
{
$field->value = $field->default_value;
}
Expand Down Expand Up @@ -358,9 +362,7 @@ function categoryHasChanged(element) {
$fieldsNode->setAttribute('name', 'com_fields');

// Organizing the fields according to their group
$fieldsPerGroup = array(
0 => array()
);
$fieldsPerGroup = array(0 => array());

foreach ($fields as $field)
{
Expand Down Expand Up @@ -479,9 +481,7 @@ function categoryHasChanged(element) {
// Loading the XML fields string into the form
$form->load($xml->saveXML());

$model = JModelLegacy::getInstance('Field', 'FieldsModel', array(
'ignore_request' => true)
);
$model = JModelLegacy::getInstance('Field', 'FieldsModel', array('ignore_request' => true));

if ((!isset($data->id) || !$data->id) && JFactory::getApplication()->input->getCmd('controller') == 'config.display.modules'
&& JFactory::getApplication()->isClient('site'))
Expand All @@ -498,7 +498,7 @@ function categoryHasChanged(element) {

foreach ($fields as $field)
{
$value = $model->getFieldValue($field->id, $field->context, $data->id);
$value = $model->getFieldValue($field->id, $data->id);

if ($value === null)
{
Expand Down
64 changes: 52 additions & 12 deletions administrator/components/com_fields/models/field.php
Original file line number Diff line number Diff line change
Expand Up @@ -417,15 +417,14 @@ public function getForm($data = array(), $loadData = true)
* Setting the value for the gven field id, context and item id.
*
* @param string $fieldId The field ID.
* @param string $context The context.
* @param string $itemId The ID of the item.
* @param string $value The value.
*
* @return boolean
*
* @since 3.7.0
*/
public function setFieldValue($fieldId, $context, $itemId, $value)
public function setFieldValue($fieldId, $itemId, $value)
{
$field = $this->getItem($fieldId);
$params = $field->params;
Expand All @@ -452,7 +451,7 @@ public function setFieldValue($fieldId, $context, $itemId, $value)
}
else
{
$oldValue = $this->getFieldValue($fieldId, $context, $itemId);
$oldValue = $this->getFieldValue($fieldId, $itemId);
$value = (array) $value;

if ($oldValue === null)
Expand Down Expand Up @@ -481,7 +480,6 @@ public function setFieldValue($fieldId, $context, $itemId, $value)

$query->delete($query->qn('#__fields_values'))
->where($query->qn('field_id') . ' = ' . (int) $fieldId)
->where($query->qn('context') . ' = ' . $query->q($context))
->where($query->qn('item_id') . ' = ' . $query->q($itemId));

$this->getDbo()->setQuery($query)->execute();
Expand All @@ -492,7 +490,6 @@ public function setFieldValue($fieldId, $context, $itemId, $value)
$newObj = new stdClass;

$newObj->field_id = (int) $fieldId;
$newObj->context = $context;
$newObj->item_id = $itemId;

foreach ($value as $v)
Expand All @@ -508,11 +505,10 @@ public function setFieldValue($fieldId, $context, $itemId, $value)
$updateObj = new stdClass;

$updateObj->field_id = (int) $fieldId;
$updateObj->context = $context;
$updateObj->item_id = $itemId;
$updateObj->value = reset($value);

$this->getDbo()->updateObject('#__fields_values', $updateObj, array('field_id', 'context', 'item_id'));
$this->getDbo()->updateObject('#__fields_values', $updateObj, array('field_id', 'item_id'));
}

$this->valueCache = array();
Expand All @@ -524,16 +520,15 @@ public function setFieldValue($fieldId, $context, $itemId, $value)
* Returning the value for the given field id, context and item id.
*
* @param string $fieldId The field ID.
* @param string $context The context.
* @param string $itemId The ID of the item.
*
* @return NULL|string
*
* @since 3.7.0
*/
public function getFieldValue($fieldId, $context, $itemId)
public function getFieldValue($fieldId, $itemId)
{
$key = md5($fieldId . $context . $itemId);
$key = md5($fieldId . $itemId);

if (!key_exists($key, $this->valueCache))
{
Expand All @@ -544,7 +539,6 @@ public function getFieldValue($fieldId, $context, $itemId)
$query->select($query->qn('value'))
->from($query->qn('#__fields_values'))
->where($query->qn('field_id') . ' = ' . (int) $fieldId)
->where($query->qn('context') . ' = ' . $query->q($context))
->where($query->qn('item_id') . ' = ' . $query->q($itemId));

$rows = $this->getDbo()->setQuery($query)->loadObjectList();
Expand All @@ -569,6 +563,46 @@ public function getFieldValue($fieldId, $context, $itemId)
return $this->valueCache[$key];
}

/**
* Returning the values for the given field ids, context and item id.
*
* @param array $fieldIds The field Ids.
* @param string $itemId The ID of the item.
*
* @return NULL|array
*
* @since 3.7.0
*/
public function getFieldValues(array $fieldIds, $itemId)
{
$key = md5(serialize($fieldIds) . $itemId);

if (!key_exists($key, $this->valueCache))
{
$this->valueCache[$key] = null;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I read the code correctly, this line is not needed. You set $this->valueCache[$key] = $data; later already

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

True


$query = $this->getDbo()->getQuery(true);

$query->select(array($query->qn('field_id'), $query->qn('value')))
->from($query->qn('#__fields_values'))
->where($query->qn('field_id') . ' in (' . implode(',', ArrayHelper::toInteger($fieldIds)) . ')')
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should be IN instead of in. Am I correct that we always use upper case string for SQL keywords?

->where($query->qn('item_id') . ' = ' . $query->q($itemId));

$rows = $this->getDbo()->setQuery($query)->loadObjectList();

$data = array();

foreach ($rows as $row)
{
$data[$row->field_id] = $row->value;
}

$this->valueCache[$key] = $data;
}

return $this->valueCache[$key];
}

/**
* Cleaning up the values for the given item on the context.
*
Expand All @@ -581,10 +615,16 @@ public function getFieldValue($fieldId, $context, $itemId)
*/
public function cleanupValues($context, $itemId)
{
// Delete with inner join is not possible so we need to do a subquery
$fieldsQuery = $this->getDbo()->getQuery(true);
$fieldsQuery->select($fieldsQuery->qn('id'))
->from($fieldsQuery->qn('#__fields'))
->where($fieldsQuery->qn('context') . ' = ' . $fieldsQuery->q($context));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks wrong to me as you have removed context from the db table ;)

Copy link
Member Author

@laoneo laoneo Mar 13, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That query is against #__fields and not #__fields_values. It is correct.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah thanks i got confused. 👍

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you know if we use in the core a delete with an inner join as subselects are slower? I couldn't find a way to do it.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know that we use it. But that does not mean that we don't use it. 😄

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not that concerned about performance for deleting fields. It's not done multiple times per second usually. Much more important that it works reliably 😄

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I saw that, but also doesn't use an inner join.


$query = $this->getDbo()->getQuery(true);

$query->delete($query->qn('#__fields_values'))
->where($query->qn('context') . ' = ' . $query->q($context))
->where($query->qn('field_id') . ' IN (' . $fieldsQuery . ')')
->where($query->qn('item_id') . ' = ' . $query->q($itemId));

$this->getDbo()->setQuery($query)->execute();
Expand Down
2 changes: 1 addition & 1 deletion administrator/components/com_fields/models/fields.php
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ protected function getStoreId($id = '')
$id .= ':' . serialize($this->getState('filter.assigned_cat_ids'));
$id .= ':' . $this->getState('filter.state');
$id .= ':' . $this->getState('filter.group_id');
$id .= ':' . print_r($this->getState('filter.language'), true);
$id .= ':' . serialize($this->getState('filter.language'));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

serialize? :-)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why not, it is an array and serialize is also used for the assigned cats.


return parent::getStoreId($id);
}
Expand Down
2 changes: 0 additions & 2 deletions installation/sql/mysql/joomla.sql
Original file line number Diff line number Diff line change
Expand Up @@ -739,11 +739,9 @@ CREATE TABLE IF NOT EXISTS `#__fields_groups` (

CREATE TABLE IF NOT EXISTS `#__fields_values` (
`field_id` int(10) unsigned NOT NULL,
`context` varchar(255) NOT NULL,
`item_id` varchar(255) NOT NULL COMMENT 'Allow references to items which have strings as ids, eg. none db systems.',
`value` text NOT NULL DEFAULT '',
KEY (`field_id`),
KEY (`context`),
KEY (`item_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_unicode_ci;

Expand Down
2 changes: 0 additions & 2 deletions installation/sql/postgresql/joomla.sql
Original file line number Diff line number Diff line change
Expand Up @@ -747,11 +747,9 @@ CREATE INDEX "#__fields_groups_idx_language" ON "#__fields_groups" ("language");
CREATE TABLE "#__fields_values" (
"field_id" bigint DEFAULT 0 NOT NULL,
"item_id" varchar(255) DEFAULT '' NOT NULL,
"context" varchar(255) DEFAULT '' NOT NULL,
"value" text DEFAULT '' NOT NULL
);
CREATE INDEX "#__fields_values_idx_field_id" ON "#__fields_values" ("field_id");
CREATE INDEX "#__fields_values_idx_context" ON "#__fields_values" ("context");
CREATE INDEX "#__fields_values_idx_item_id" ON "#__fields_values" ("item_id");

--
Expand Down
5 changes: 0 additions & 5 deletions installation/sql/sqlazure/joomla.sql
Original file line number Diff line number Diff line change
Expand Up @@ -999,7 +999,6 @@ WITH (STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, O

CREATE TABLE "#__fields_values" (
"field_id" bigint NOT NULL DEFAULT 1,
"context" nvarchar(255) NOT NULL DEFAULT '',
"item_id" nvarchar(255) NOT NULL DEFAULT '',
"value" nvarchar(max) NOT NULL DEFAULT '',
) ON [PRIMARY];
Expand All @@ -1008,10 +1007,6 @@ CREATE NONCLUSTERED INDEX "idx_field_id" ON "#__fields_values" (
"field_id" ASC)
WITH (STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF);

CREATE NONCLUSTERED INDEX "idx_context" ON "#__fields_values" (
"context" ASC)
WITH (STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF);

CREATE NONCLUSTERED INDEX "idx_item_id" ON "#__fields_values" (
"item_id" ASC)
WITH (STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF);
Expand Down
2 changes: 1 addition & 1 deletion plugins/system/fields/fields.php
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ public function onContentAfterSave($context, $item, $isNew, $data = array())
}

// Setting the value for the field and the item
$model->setFieldValue($field->id, $context, $id, $fieldsData[$field->alias]);
$model->setFieldValue($field->id, $id, $fieldsData[$field->alias]);
}

return true;
Expand Down