Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[INLONG-8047][Manager][Sort] Rename lightweight to dataSync #8048

Merged
merged 5 commits into from
May 22, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions inlong-dashboard/src/ui/pages/GroupDashboard/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ const Comp: React.FC = () => {
// mqType: '',
pageSize: defaultSize,
pageNum: 1,
lightweight: 0,
inlongGroupMode: 0,
});

const [groupLogs, setGroupLogs] = useState({
Expand All @@ -52,7 +52,7 @@ const Comp: React.FC = () => {
const { data: summary = {} } = useRequest({
url: '/group/countByStatus',
params: {
lightweight: 0,
inlongGroupMode: 0,
},
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,10 @@ public InlongGroupInfo createGroupInfo() {
pulsarInfo.setTenant(tenant);
pulsarInfo.setMqResource(namespace);

// set enable zk, create resource, lightweight mode, and cluster tag
// set enable zk, create resource, group mode, and cluster tag
pulsarInfo.setEnableZookeeper(InlongConstants.DISABLE_ZK);
pulsarInfo.setEnableCreateResource(InlongConstants.ENABLE_CREATE_RESOURCE);
pulsarInfo.setLightweight(InlongConstants.STANDARD_MODE);
pulsarInfo.setInlongGroupMode(InlongConstants.STANDARD_MODE);
pulsarInfo.setInlongClusterTag("default_cluster");

pulsarInfo.setDailyRecords(10000000);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -106,10 +106,10 @@ public static InlongGroupInfo createGroupInfo() {
pulsarInfo.setTenant(TENANT);
pulsarInfo.setMqResource(NAMESPACE);

// set enable zk, create resource, lightweight mode, and cluster tag
// set enable zk, create resource, group mode, and cluster tag
pulsarInfo.setEnableZookeeper(InlongConstants.DISABLE_ZK);
pulsarInfo.setEnableCreateResource(InlongConstants.ENABLE_CREATE_RESOURCE);
pulsarInfo.setLightweight(InlongConstants.LIGHTWEIGHT_MODE);
pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_MODE);
pulsarInfo.setInlongClusterTag("default_cluster");

pulsarInfo.setDailyRecords(10000000);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ static void createStub() {
"{\"formName\":\"ApplyGroupProcessForm\",\"groupInfo\":{\"mqType\":\"PULSAR\",\"id\":6,"
+ "\"inlongGroupId\":\"test_group009\",\"name\":null,\"description\":null,"
+ "\"mqResource\":\"test_namespace\",\"enableZookeeper\":0,"
+ "\"enableCreateResource\":1,\"lightweight\":1,"
+ "\"enableCreateResource\":1,\"inlongGroupMode\":1,"
+ "\"inlongClusterTag\":\"default_cluster\",\"dailyRecords\":10000000,"
+ "\"dailyStorage\":10000,\"peakRecords\":100000,\"maxLength\":10000,"
+ "\"inCharges\":\"test_inCharges,admin\",\"followers\":null,\"status\":101,"
Expand Down Expand Up @@ -161,7 +161,7 @@ static void createStub() {
.formData("{\"formName\":\"ApplyGroupProcessForm\",\"groupInfo\":{\"mqType\":\"PULSAR\","
+ "\"id\":8,\"inlongGroupId\":\"test_group011\",\"name\":null,\"description\":null,"
+ "\"mqResource\":\"test_namespace\",\"enableZookeeper\":0,\"enableCreateResource\":1,"
+ "\"lightweight\":1,\"inlongClusterTag\":\"default_cluster\","
+ "\"inlongGroupMode\":1,\"inlongClusterTag\":\"default_cluster\","
+ "\"dailyRecords\":10000000,\"dailyStorage\":10000,\"peakRecords\":100000,"
+ "\"maxLength\":10000,\"inCharges\":\"test_inCharges,admin\",\"followers\":null,"
+ "\"status\":101,\"creator\":\"admin\",\"modifier\":\"admin\","
Expand Down Expand Up @@ -189,7 +189,7 @@ static void createStub() {
pulsarInfo.setMqResource("test_namespace");
pulsarInfo.setEnableZookeeper(0);
pulsarInfo.setEnableCreateResource(1);
pulsarInfo.setLightweight(1);
pulsarInfo.setInlongGroupMode(1);
pulsarInfo.setInlongClusterTag("default_cluster");
pulsarInfo.setInCharges("test_inCharges,admin");
pulsarInfo.setStatus(130);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
import static org.apache.inlong.manager.client.cli.consts.GroupConstants.DEFAULT_DATA_ENCODING;
import static org.apache.inlong.manager.client.cli.consts.GroupConstants.DEFAULT_DATA_SEPARATOR;
import static org.apache.inlong.manager.client.cli.consts.GroupConstants.DEFAULT_IGNORE_PARSE_ERROR;
import static org.apache.inlong.manager.client.cli.consts.GroupConstants.DEFAULT_LIGHTWEIGHT;
import static org.apache.inlong.manager.client.cli.consts.GroupConstants.DEFAULT_INLONG_GROUP_MODE;
import static org.apache.inlong.manager.common.consts.InlongConstants.ADMIN_USER;

/**
Expand Down Expand Up @@ -117,7 +117,7 @@ private void setDefaultConfigInfo(CreateGroupConf groupConf) {
String inlongStreamId = groupConf.getStreamInfo().getInlongStreamId();
// group
groupConf.getGroupInfo().setInCharges(ADMIN_USER);
groupConf.getGroupInfo().setLightweight(DEFAULT_LIGHTWEIGHT);
groupConf.getGroupInfo().setInlongGroupMode(DEFAULT_INLONG_GROUP_MODE);
groupConf.getGroupInfo().setSortConf(new FlinkSortConf());

// stream
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

public class GroupConstants {

public static final int DEFAULT_LIGHTWEIGHT = 0;
public static final int DEFAULT_INLONG_GROUP_MODE = 0;

public static final String DEFAULT_DATA_ENCODING = "UTF-8";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ void testParseForm() {
+ " \"mqResource\" : \"test_namespace\",\n"
+ " \"enableZookeeper\" : 0,\n"
+ " \"enableCreateResource\" : 1,\n"
+ " \"lightweight\" : 1,\n"
+ " \"inlongGroupMode\" : 1,\n"
+ " \"inlongClusterTag\" : \"default_cluster\",\n"
+ " \"dailyRecords\" : 10000000,\n"
+ " \"dailyStorage\" : 10000,\n"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public class InlongConstants {
public static final Integer DELETED_STATUS = 10;

public static final Integer STANDARD_MODE = 0;
public static final Integer LIGHTWEIGHT_MODE = 1;
public static final Integer DATASYNC_MODE = 1;

public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public enum ErrorCodeEnum {
GROUP_MIDDLEWARE_UPDATE_NOT_ALLOWED(1011, "The current status does not support modifying the MQ type"),
GROUP_NAME_UPDATE_NOT_ALLOWED(1012, "The current inlong group status does not support modifying the name"),
GROUP_INFO_INCONSISTENT(1013, "The inlong group info is inconsistent, please contact the administrator"),
GROUP_MODE_UNSUPPORTED(1014, "The current inlong group mode only support lightweight or standard"),
GROUP_MODE_UNSUPPORTED(1014, "The current inlong group mode only support dataSync or standard"),

OPT_NOT_ALLOWED_BY_STATUS(1021, "InlongGroup status %s was not allowed to add/update/delete related info"),

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,10 @@ public enum GroupMode {
STANDARD("standard"),

/**
* Lightweight group init with sort in Inlong Cluster
* DataSync group init with sort in Inlong Cluster
* StreamSource -> Sort -> StreamSink
*/
LIGHTWEIGHT("lightweight");
DATASYNC("datasync");

@Getter
private final String mode;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public class InlongGroupEntity implements Serializable {

private Integer enableZookeeper;
private Integer enableCreateResource;
private Integer lightweight;
private Integer inlongGroupMode;
private Integer dataReportType;
private String inlongClusterTag;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public interface InlongGroupEntityMapper {
InlongGroupEntity selectByPrimaryKey(Integer id);

List<Map<String, Object>> countGroupByUser(@Param(value = "username") String username,
@Param(value = "lightweight") Integer lightweight);
@Param(value = "inlongGroupMode") Integer inlongGroupMode);

InlongGroupEntity selectByGroupId(String groupId);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
<result column="max_length" jdbcType="INTEGER" property="maxLength"/>
<result column="enable_zookeeper" jdbcType="INTEGER" property="enableZookeeper"/>
<result column="enable_create_resource" jdbcType="INTEGER" property="enableCreateResource"/>
<result column="lightweight" jdbcType="INTEGER" property="lightweight"/>
<result column="inlong_group_mode" jdbcType="INTEGER" property="inlongGroupMode"/>
<result column="data_report_type" jdbcType="INTEGER" property="dataReportType"/>
<result column="inlong_cluster_tag" jdbcType="VARCHAR" property="inlongClusterTag"/>
<result column="ext_params" jdbcType="LONGVARCHAR" property="extParams"/>
Expand All @@ -57,7 +57,7 @@

<sql id="Base_Column_List">
id, inlong_group_id, name, description, mq_type, mq_resource, daily_records, daily_storage,
peak_records, max_length, enable_zookeeper, enable_create_resource, lightweight, data_report_type,
peak_records, max_length, enable_zookeeper, enable_create_resource, inlong_group_mode, data_report_type,
inlong_cluster_tag, ext_params, in_charges, followers, status, previous_status, is_deleted,
creator, modifier, create_time, modify_time, version
</sql>
Expand All @@ -70,7 +70,7 @@
daily_records, daily_storage,
peak_records, max_length,
enable_zookeeper, enable_create_resource,
lightweight, data_report_type,
inlong_group_mode, data_report_type,
inlong_cluster_tag, ext_params,
in_charges, followers,
status, previous_status,
Expand All @@ -81,7 +81,7 @@
#{dailyRecords,jdbcType=INTEGER}, #{dailyStorage,jdbcType=INTEGER},
#{peakRecords,jdbcType=INTEGER}, #{maxLength,jdbcType=INTEGER},
#{enableZookeeper,jdbcType=INTEGER}, #{enableCreateResource,jdbcType=INTEGER},
#{lightweight,jdbcType=INTEGER}, #{dataReportType,jdbcType=INTEGER},
#{inlongGroupMode,jdbcType=INTEGER}, #{dataReportType,jdbcType=INTEGER},
#{inlongClusterTag,jdbcType=VARCHAR}, #{extParams,jdbcType=LONGVARCHAR},
#{inCharges,jdbcType=VARCHAR}, #{followers,jdbcType=VARCHAR},
#{status,jdbcType=INTEGER}, #{previousStatus,jdbcType=INTEGER},
Expand All @@ -98,8 +98,8 @@
select count(*) as total, status
from inlong_group
where is_deleted = 0
<if test="lightweight != null">
and lightweight = #{lightweight,jdbcType=TINYINT}
<if test="inlongGroupMode != null">
and inlong_group_mode = #{inlongGroupMode,jdbcType=TINYINT}
</if>
and (creator = #{username,jdbcType=VARCHAR} or find_in_set(#{username,jdbcType=VARCHAR}, in_charges))
group by status
Expand Down Expand Up @@ -157,8 +157,8 @@
#{tag}
</foreach>
</if>
<if test="lightweight != null">
and lightweight = #{lightweight,jdbcType=TINYINT}
<if test="inlongGroupMode != null">
and inlong_group_mode = #{inlongGroupMode,jdbcType=TINYINT}
</if>
</where>
<choose>
Expand Down Expand Up @@ -251,7 +251,7 @@

enable_zookeeper = #{enableZookeeper,jdbcType=INTEGER},
enable_create_resource = #{enableCreateResource,jdbcType=INTEGER},
lightweight = #{lightweight,jdbcType=INTEGER},
inlong_group_mode = #{inlongGroupMode,jdbcType=INTEGER},
data_report_type = #{dataReportType,jdbcType=INTEGER},
inlong_cluster_tag = #{inlongClusterTag,jdbcType=VARCHAR},

Expand Down Expand Up @@ -299,8 +299,8 @@
<if test="enableCreateResource != null">
enable_create_resource = #{enableCreateResource,jdbcType=INTEGER},
</if>
<if test="lightweight != null">
lightweight = #{lightweight,jdbcType=INTEGER},
<if test="inlongGroupMode != null">
inlong_group_mode = #{inlongGroupMode,jdbcType=INTEGER},
</if>
<if test="dataReportType != null">
data_report_type = #{dataReportType,jdbcType=INTEGER},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@ public abstract class InlongGroupInfo extends BaseInlongGroup {
@ApiModelProperty(value = "Whether to enable create resource? 0: disable, 1: enable")
private Integer enableCreateResource = 0;

@ApiModelProperty(value = "Whether to use lightweight mode, 0: no, 1: yes")
private Integer lightweight;
@ApiModelProperty(value = "Standard mode: 0, DataSync mode: 1")
private Integer inlongGroupMode;

@ApiModelProperty(value = "Data report type, default is 0.\n"
+ " 0: report to DataProxy and respond when the DataProxy received data.\n"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,8 @@ public class InlongGroupPageRequest extends PageRequest {
@ApiModelProperty(value = "The inlong cluster tag list")
private List<String> clusterTagList;

@ApiModelProperty(value = "Standard mode: 0, Lightweight: 1")
private Integer lightweight;
@ApiModelProperty(value = "Standard mode: 0, DataSync mode: 1")
private Integer inlongGroupMode;

@ApiModelProperty(value = "Current user", hidden = true)
private String currentUser;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,9 @@ public abstract class InlongGroupRequest extends BaseInlongGroup {
@Range(min = 0, max = 1, message = "default is 1, only supports [0: disable, 1: enable]")
private Integer enableCreateResource = 1;

@ApiModelProperty(value = "Whether to use lightweight mode, 0: no, 1: yes")
@Range(min = 0, max = 1, message = "default is 0, only supports [0: no, 1: yes]")
private Integer lightweight = 0;
@ApiModelProperty(value = "Standard mode: 0, DataSync mode: 1")
@Range(min = 0, max = 1, message = "default is 0, only supports [0: Standard, 1: DataSync]")
private Integer inlongGroupMode = 0;

@ApiModelProperty(value = "Data report type, default is 0.\n"
+ " 0: report to DataProxy and respond when the DataProxy received data.\n"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,13 @@ void testDeserializePageInfoFromWeb() {

String pageInfoJson = "{\"total\":30,\"list\":[{\"id\":null,\"inlongGroupId\":\"22\",\"name\":null,"
+ "\"description\":null,\"middlewareType\":null,\"mqType\":\"NONE\",\"mqResource\":null,"
+ "\"enableZookeeper\":0,\"enableCreateResource\":null,\"lightweight\":null,"
+ "\"enableZookeeper\":0,\"enableCreateResource\":null,\"inlongGroupMode\":null,"
+ "\"inlongClusterTag\":null,\"dailyRecords\":null,\"dailyStorage\":null,\"peakRecords\":null,"
+ "\"maxLength\":null,\"inCharges\":null,\"followers\":null,\"status\":null,\"creator\":null,"
+ "\"modifier\":null,\"createTime\":null,\"modifyTime\":null,\"extList\":null,\"sortConf\":null,"
+ "\"version\":44},{\"id\":null,\"inlongGroupId\":null,\"name\":null,\"description\":null,"
+ "\"middlewareType\":null,\"mqType\":\"PULSAR\",\"mqResource\":null,\"enableZookeeper\":0,"
+ "\"enableCreateResource\":null,\"lightweight\":null,\"inlongClusterTag\":null,"
+ "\"enableCreateResource\":null,\"inlongGroupMode\":null,\"inlongClusterTag\":null,"
+ "\"dailyRecords\":null,\"dailyStorage\":null,\"peakRecords\":null,\"maxLength\":null,"
+ "\"inCharges\":null,\"followers\":null,\"status\":null,\"creator\":null,\"modifier\":null,"
+ "\"createTime\":null,\"modifyTime\":null,\"extList\":null,\"sortConf\":null,\"version\":null,"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ String save(@Valid @NotNull(message = "inlong group request cannot be null") Inl
* @param operator name of operator
* @return inlong group status statistics
*/
InlongGroupCountResponse countGroupByUser(String operator, Integer lightweight);
InlongGroupCountResponse countGroupByUser(String operator, Integer inlongGroupMode);

/**
* According to the group id, query the topic to which it belongs
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -244,9 +244,9 @@ public InlongGroupInfo get(String groupId, UserInfo opInfo) {
}

@Override
public InlongGroupCountResponse countGroupByUser(String operator, Integer lightweight) {
public InlongGroupCountResponse countGroupByUser(String operator, Integer inlongGroupMode) {
InlongGroupCountResponse countVO = new InlongGroupCountResponse();
List<Map<String, Object>> statusCount = groupMapper.countGroupByUser(operator, lightweight);
List<Map<String, Object>> statusCount = groupMapper.countGroupByUser(operator, inlongGroupMode);
for (Map<String, Object> map : statusCount) {
int status = (Integer) map.get("status");
long count = (Long) map.get("count");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ public ListenerResult listen(WorkflowContext context) throws WorkflowListenerExc

// update status of other related configs
if (InlongConstants.DISABLE_CREATE_RESOURCE.equals(groupInfo.getEnableCreateResource())) {
if (InlongConstants.LIGHTWEIGHT_MODE.equals(groupInfo.getLightweight())) {
if (InlongConstants.DATASYNC_MODE.equals(groupInfo.getInlongGroupMode())) {
sourceService.updateStatus(groupId, null, SourceStatus.SOURCE_NORMAL.getCode(), operator);
} else {
sourceService.updateStatus(groupId, null, SourceStatus.TO_BE_ISSUED_ADD.getCode(), operator);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,16 +80,16 @@ public ListenerResult listen(WorkflowContext context) {
break;
}

// if the inlong group is lightweight mode, the stream source needs to be processed.
if (InlongConstants.LIGHTWEIGHT_MODE.equals(groupInfo.getLightweight())) {
changeSource4Lightweight(groupId, operateType, operator);
// if the inlong group is dataSync mode, the stream source needs to be processed.
if (InlongConstants.DATASYNC_MODE.equals(groupInfo.getInlongGroupMode())) {
changeSource4DataSync(groupId, operateType, operator);
}

log.info("success to execute UpdateGroupCompleteListener for groupId={}, operateType={}", groupId, operateType);
return ListenerResult.success();
}

private void changeSource4Lightweight(String groupId, GroupOperateType operateType, String operator) {
private void changeSource4DataSync(String groupId, GroupOperateType operateType, String operator) {
switch (operateType) {
case SUSPEND:
sourceService.updateStatus(groupId, null, SourceStatus.SOURCE_STOP.getCode(), operator);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ public boolean accept(WorkflowContext context) {
return false;
}
GroupResourceProcessForm processForm = (GroupResourceProcessForm) context.getProcessForm();
return InlongConstants.STANDARD_MODE.equals(processForm.getGroupInfo().getLightweight());
return InlongConstants.STANDARD_MODE.equals(processForm.getGroupInfo().getInlongGroupMode());
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public boolean accept(WorkflowContext context) {
return false;
}
StreamResourceProcessForm streamProcessForm = (StreamResourceProcessForm) processForm;
return InlongConstants.STANDARD_MODE.equals(streamProcessForm.getGroupInfo().getLightweight());
return InlongConstants.STANDARD_MODE.equals(streamProcessForm.getGroupInfo().getInlongGroupMode());
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ public boolean accept(WorkflowContext context) {
return false;
}
GroupResourceProcessForm processForm = (GroupResourceProcessForm) context.getProcessForm();
return InlongConstants.STANDARD_MODE.equals(processForm.getGroupInfo().getLightweight())
return InlongConstants.STANDARD_MODE.equals(processForm.getGroupInfo().getInlongGroupMode())
&& processForm.getGroupOperateType() == GroupOperateType.DELETE;
}

Expand Down
Loading