Skip to content

Commit

Permalink
Fix ignored bitmap of attribute in writeTablet (apache#13858)
Browse files Browse the repository at this point in the history
* Fix ignored bitmap of attribute in writeTablet

* add license

* fix index
  • Loading branch information
jt2594838 authored Oct 22, 2024
1 parent 57e450c commit aea9f3a
Show file tree
Hide file tree
Showing 4 changed files with 164 additions and 23 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public PipeConvertedInsertTabletStatement(final InsertTabletStatement insertTabl
dataTypes = insertTabletStatement.getDataTypes();
// InsertTabletStatement
times = insertTabletStatement.getTimes();
bitMaps = insertTabletStatement.getBitMaps();
nullBitMaps = insertTabletStatement.getBitMaps();
columns = insertTabletStatement.getColumns();
rowCount = insertTabletStatement.getRowCount();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,11 +78,14 @@ public List<Object[]> getAttributeValueList() {
final InsertTabletStatement insertTabletStatement = getInnerTreeStatement();
List<Object[]> result = new ArrayList<>(insertTabletStatement.getRowCount());
final List<Integer> attrColumnIndices = insertTabletStatement.getAttrColumnIndices();
for (int i = 0; i < insertTabletStatement.getRowCount(); i++) {
for (int rowIndex = 0; rowIndex < insertTabletStatement.getRowCount(); rowIndex++) {
Object[] attrValues = new Object[attrColumnIndices.size()];
for (int j = 0; j < attrColumnIndices.size(); j++) {
final int columnIndex = attrColumnIndices.get(j);
attrValues[j] = ((Object[]) insertTabletStatement.getColumns()[columnIndex])[i];
for (int attrColNum = 0; attrColNum < attrColumnIndices.size(); attrColNum++) {
final int columnIndex = attrColumnIndices.get(attrColNum);
if (!insertTabletStatement.isNull(rowIndex, columnIndex)) {
attrValues[attrColNum] =
((Object[]) insertTabletStatement.getColumns()[columnIndex])[rowIndex];
}
}
result.add(attrValues);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
private static final String DATATYPE_UNSUPPORTED = "Data type %s is not supported.";

protected long[] times; // times should be sorted. It is done in the session API.
protected BitMap[] bitMaps;
protected BitMap[] nullBitMaps;
protected Object[] columns;

private IDeviceID[] deviceIDs;
Expand Down Expand Up @@ -122,11 +122,11 @@ public void setColumns(Object[] columns) {
}

public BitMap[] getBitMaps() {
return bitMaps;
return nullBitMaps;
}

public void setBitMaps(BitMap[] bitMaps) {
this.bitMaps = bitMaps;
this.nullBitMaps = bitMaps;
}

public long[] getTimes() {
Expand Down Expand Up @@ -278,8 +278,8 @@ public List<InsertTabletStatement> getSplitList() {
measurements[i] = pairList.get(i).left;
measurementSchemas[i] = this.measurementSchemas[realIndex];
dataTypes[i] = this.dataTypes[realIndex];
if (this.bitMaps != null) {
copiedBitMaps[i] = this.bitMaps[realIndex];
if (this.nullBitMaps != null) {
copiedBitMaps[i] = this.nullBitMaps[realIndex];
}
if (this.measurementIsAligned != null) {
statement.setAligned(this.measurementIsAligned[realIndex]);
Expand All @@ -289,7 +289,7 @@ public List<InsertTabletStatement> getSplitList() {
statement.setMeasurements(measurements);
statement.setMeasurementSchemas(measurementSchemas);
statement.setDataTypes(dataTypes);
if (this.bitMaps != null) {
if (this.nullBitMaps != null) {
statement.setBitMaps(copiedBitMaps);
}
statement.setFailedMeasurementIndex2Info(failedMeasurementIndex2Info);
Expand Down Expand Up @@ -461,8 +461,9 @@ public IDeviceID getTableDeviceID(int rowIdx) {
deviceIdSegments[0] = this.getTableName();
for (int i = 0; i < getIdColumnIndices().size(); i++) {
final Integer columnIndex = getIdColumnIndices().get(i);
Object idSeg = ((Object[]) columns[columnIndex])[rowIdx];
deviceIdSegments[i + 1] = idSeg != null ? idSeg.toString() : null;
boolean isNull = isNull(rowIdx, i);
deviceIdSegments[i + 1] =
isNull ? null : ((Object[]) columns[columnIndex])[rowIdx].toString();
}
deviceIDs[rowIdx] = Factory.DEFAULT_FACTORY.create(deviceIdSegments);
}
Expand All @@ -474,21 +475,21 @@ public IDeviceID getTableDeviceID(int rowIdx) {
public void insertColumn(int pos, ColumnSchema columnSchema) {
super.insertColumn(pos, columnSchema);

if (bitMaps == null) {
bitMaps = new BitMap[measurements.length];
bitMaps[pos] = new BitMap(rowCount);
if (nullBitMaps == null) {
nullBitMaps = new BitMap[measurements.length];
nullBitMaps[pos] = new BitMap(rowCount);
for (int i = 0; i < rowCount; i++) {
bitMaps[pos].mark(i);
nullBitMaps[pos].mark(i);
}
} else {
BitMap[] tmpBitmaps = new BitMap[bitMaps.length + 1];
System.arraycopy(bitMaps, 0, tmpBitmaps, 0, pos);
BitMap[] tmpBitmaps = new BitMap[nullBitMaps.length + 1];
System.arraycopy(nullBitMaps, 0, tmpBitmaps, 0, pos);
tmpBitmaps[pos] = new BitMap(rowCount);
for (int i = 0; i < rowCount; i++) {
tmpBitmaps[pos].mark(i);
}
System.arraycopy(bitMaps, pos, tmpBitmaps, pos + 1, bitMaps.length - pos);
bitMaps = tmpBitmaps;
System.arraycopy(nullBitMaps, pos, tmpBitmaps, pos + 1, nullBitMaps.length - pos);
nullBitMaps = tmpBitmaps;
}

Object[] tmpColumns = new Object[columns.length + 1];
Expand All @@ -507,10 +508,17 @@ public void insertColumn(int pos, ColumnSchema columnSchema) {
@Override
public void swapColumn(int src, int target) {
super.swapColumn(src, target);
if (bitMaps != null) {
CommonUtils.swapArray(bitMaps, src, target);
if (nullBitMaps != null) {
CommonUtils.swapArray(nullBitMaps, src, target);
}
CommonUtils.swapArray(columns, src, target);
deviceIDs = null;
}

public boolean isNull(int row, int col) {
if (nullBitMaps == null || nullBitMaps[col] == null) {
return false;
}
return nullBitMaps[col].isMarked(row);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;

import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;

import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
import org.junit.Test;

import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;

public class InsertTabletTest {

@Test
public void testWithNull() {
InsertTabletStatement innerStmt = new InsertTabletStatement();
innerStmt.setDevicePath(new PartialPath("table1", false));
innerStmt.setTimes(new long[] {1, 2, 3, 4});
innerStmt.setRowCount(4);
innerStmt.setMeasurements(
new String[] {"deviceId_1", "deviceId_2", "attr1", "attr2", "measurement"});
innerStmt.setColumnCategories(
new TsTableColumnCategory[] {
TsTableColumnCategory.ID, TsTableColumnCategory.ID,
TsTableColumnCategory.ATTRIBUTE, TsTableColumnCategory.ATTRIBUTE,
TsTableColumnCategory.MEASUREMENT
});
innerStmt.setDataTypes(
new TSDataType[] {
TSDataType.STRING,
TSDataType.STRING,
TSDataType.STRING,
TSDataType.STRING,
TSDataType.STRING
});
innerStmt.setColumns(
new Object[] {
new Binary[] {
new Binary("id1_1", StandardCharsets.UTF_8),
Binary.EMPTY_VALUE,
new Binary("id3_1", StandardCharsets.UTF_8),
Binary.EMPTY_VALUE,
},
new Binary[] {
new Binary("id1_2", StandardCharsets.UTF_8),
new Binary("id2_2", StandardCharsets.UTF_8),
Binary.EMPTY_VALUE,
Binary.EMPTY_VALUE,
},
new Binary[] {
new Binary("attr1_1", StandardCharsets.UTF_8),
new Binary("attr2_1", StandardCharsets.UTF_8),
Binary.EMPTY_VALUE,
Binary.EMPTY_VALUE,
},
new Binary[] {
new Binary("attr1_2", StandardCharsets.UTF_8),
Binary.EMPTY_VALUE,
new Binary("attr3_2", StandardCharsets.UTF_8),
Binary.EMPTY_VALUE,
},
new Binary[] {
new Binary("m1", StandardCharsets.UTF_8),
new Binary("m2", StandardCharsets.UTF_8),
new Binary("m3", StandardCharsets.UTF_8),
new Binary("m4", StandardCharsets.UTF_8)
},
});
innerStmt.setBitMaps(
new BitMap[] {
new BitMap(4, new byte[] {1 << 1 | 1 << 3}),
new BitMap(4, new byte[] {1 << 2 | 1 << 3}),
new BitMap(4, new byte[] {1 << 2 | 1 << 3}),
new BitMap(4, new byte[] {1 << 1 | 1 << 3}),
new BitMap(4, new byte[] {0x00}),
});

InsertTablet insertTablet = new InsertTablet(innerStmt, null);
assertEquals(Arrays.asList("attr1", "attr2"), insertTablet.getAttributeColumnNameList());
List<Object[]> attributeValueList = insertTablet.getAttributeValueList();
assertEquals(4, attributeValueList.size());
assertArrayEquals(
new Object[] {
new Binary("attr1_1", StandardCharsets.UTF_8),
new Binary("attr1_2", StandardCharsets.UTF_8)
},
attributeValueList.get(0));
assertArrayEquals(
new Object[] {new Binary("attr2_1", StandardCharsets.UTF_8), null},
attributeValueList.get(1));
assertArrayEquals(
new Object[] {null, new Binary("attr3_2", StandardCharsets.UTF_8)},
attributeValueList.get(2));
assertArrayEquals(new Object[] {null, null}, attributeValueList.get(3));

List<Object[]> deviceIdList = insertTablet.getDeviceIdList();
assertEquals(4, deviceIdList.size());
assertArrayEquals(new Object[] {"id1_1", "id1_2"}, deviceIdList.get(0));
// notice: trailing nulls are removed
assertArrayEquals(new Object[] {null, "id2_2"}, deviceIdList.get(1));
assertArrayEquals(new Object[] {"id3_1"}, deviceIdList.get(2));
assertArrayEquals(new Object[] {}, deviceIdList.get(3));
}
}

0 comments on commit aea9f3a

Please sign in to comment.