Skip to content

Commit

Permalink
Merge remote-tracking branch 'apache-github/trunk' into KAFKA-13900
Browse files Browse the repository at this point in the history 
* apache-github/trunk: (447 commits)
  KAFKA-13959: Controller should unfence Broker with busy metadata log (apache#12274)
  KAFKA-10199: Expose read only task from state updater (apache#12497)
  KAFKA-14154; Return NOT_CONTROLLER from AlterPartition if leader is ahead of controller (apache#12506)
  KAFKA-13986; Brokers should include node.id in fetches to metadata quorum (apache#12498)
  KAFKA-14163; Retry compilation after zinc compile cache error (apache#12507)
  Remove duplicate common.message.* from clients:test jar file (apache#12407)
  KAFKA-13060: Replace EasyMock and PowerMock with Mockito in WorkerGroupMemberTest.java (apache#12484)
  Fix the rate window size calculation for edge cases (apache#12184)
  MINOR: Upgrade gradle to 7.5.1 and bump other build/test dependencies (apache#12495)
  KAFKA-14140: Ensure an offline or in-controlled-shutdown replica is not eligible to join ISR in ZK mode (apache#12487)
  KAFKA-14114: Add Metadata Error Related Metrics
  MINOR: BrokerMetadataSnapshotter must avoid exceeding batch size (apache#12486)
  MINOR: Upgrade mockito test dependencies (apache#12460)
  KAFKA-14144:; Compare AlterPartition LeaderAndIsr before fencing partition epoch (apache#12489)
  KAFKA-14134: Replace EasyMock with Mockito for WorkerConnectorTest (apache#12472)
  MINOR: Update scala version in bin scripts to 2.13.8 (apache#12477)
  KAFKA-14104; Add CRC validation when iterating over Metadata Log Records (apache#12457)
  MINOR: add :server-common test dependency to :storage (apache#12488)
  KAFKA-14107: Upgrade Jetty version for CVE fixes (apache#12440)
  KAFKA-14124: improve quorum controller fault handling (apache#12447)
  ...
  • Loading branch information
@ijuma
ijuma committed Aug 12, 2022
2 parents 27cb7ad + 50e5b32 commit 9a644f1
Show file tree
Hide file tree
Showing 1,193 changed files with 77,017 additions and 23,546 deletions.
38 changes: 1 addition & 37 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

def doValidation() {
sh """
./gradlew -PscalaVersion=$SCALA_VERSION clean compileJava compileScala compileTestJava compileTestScala \
./retry_zinc ./gradlew -PscalaVersion=$SCALA_VERSION clean compileJava compileScala compileTestJava compileTestScala \
spotlessScalaCheck checkstyleMain checkstyleTest spotbugsMain rat \
--profile --no-daemon --continue -PxmlSpotBugsReport=true
"""
Expand Down Expand Up @@ -160,42 +160,6 @@ pipeline {
echo 'Skipping Kafka Streams archetype test for Java 17'
}
}

stage('ARM') {
agent { label 'arm4' }
options {
timeout(time: 2, unit: 'HOURS')
timestamps()
}
environment {
SCALA_VERSION=2.12
}
steps {
doValidation()
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
doTest(env, 'unitTest')
}
echo 'Skipping Kafka Streams archetype test for ARM build'
}
}

stage('PowerPC') {
agent { label 'ppc64le' }
options {
timeout(time: 2, unit: 'HOURS')
timestamps()
}
environment {
SCALA_VERSION=2.12
}
steps {
doValidation()
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
doTest(env, 'unitTest')
}
echo 'Skipping Kafka Streams archetype test for PowerPC build'
}
}

// To avoid excessive Jenkins resource usage, we only run the stages
// above at the PR stage. The ones below are executed after changes
Expand Down
85 changes: 44 additions & 41 deletions LICENSE-binary
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,50 +208,53 @@ License Version 2.0:
audience-annotations-0.5.0
commons-cli-1.4
commons-lang3-3.8.1
jackson-annotations-2.12.3
jackson-core-2.12.3
jackson-databind-2.12.3
jackson-dataformat-csv-2.12.3
jackson-datatype-jdk8-2.12.3
jackson-jaxrs-base-2.12.3
jackson-jaxrs-json-provider-2.12.3
jackson-module-jaxb-annotations-2.12.3
jackson-annotations-2.12.6
jackson-core-2.12.6
jackson-databind-2.12.6.1
jackson-dataformat-csv-2.12.6
jackson-datatype-jdk8-2.12.6
jackson-jaxrs-base-2.12.6
jackson-jaxrs-json-provider-2.12.6
jackson-module-jaxb-annotations-2.12.6
jackson-module-paranamer-2.10.5
jackson-module-scala_2.13-2.12.3
jackson-module-scala_2.13-2.12.6
jakarta.validation-api-2.0.2
javassist-3.27.0-GA
jetty-client-9.4.43.v20210629
jetty-continuation-9.4.43.v20210629
jetty-http-9.4.43.v20210629
jetty-io-9.4.43.v20210629
jetty-security-9.4.43.v20210629
jetty-server-9.4.43.v20210629
jetty-servlet-9.4.43.v20210629
jetty-servlets-9.4.43.v20210629
jetty-util-9.4.43.v20210629
jetty-util-ajax-9.4.43.v20210629
jetty-client-9.4.44.v20210927
jetty-continuation-9.4.44.v20210927
jetty-http-9.4.44.v20210927
jetty-io-9.4.44.v20210927
jetty-security-9.4.44.v20210927
jetty-server-9.4.44.v20210927
jetty-servlet-9.4.44.v20210927
jetty-servlets-9.4.44.v20210927
jetty-util-9.4.44.v20210927
jetty-util-ajax-9.4.44.v20210927
jersey-common-2.34
jersey-server-2.34
jose4j-0.7.8
log4j-1.2.17
jose4j-0.7.9
lz4-java-1.8.0
maven-artifact-3.8.1
maven-artifact-3.8.4
metrics-core-4.1.12.1
netty-buffer-4.1.68.Final
netty-codec-4.1.68.Final
netty-common-4.1.68.Final
netty-handler-4.1.68.Final
netty-resolver-4.1.68.Final
netty-transport-4.1.68.Final
netty-transport-native-epoll-4.1.68.Final
netty-transport-native-unix-common-4.1.68.Final
plexus-utils-3.2.1
rocksdbjni-6.22.1.1
scala-collection-compat_2.13-2.4.4
scala-library-2.13.6
scala-logging_2.13-3.9.3
scala-reflect-2.13.6
scala-java8-compat_2.13-1.0.0
metrics-core-2.2.0
netty-buffer-4.1.73.Final
netty-codec-4.1.73.Final
netty-common-4.1.73.Final
netty-handler-4.1.73.Final
netty-resolver-4.1.73.Final
netty-tcnative-classes-2.0.46.Final
netty-transport-4.1.73.Final
netty-transport-classes-epoll-4.1.73.Final
netty-transport-native-epoll-4.1.73.Final
netty-transport-native-unix-common-4.1.73.Final
plexus-utils-3.3.0
reload4j-1.2.19
rocksdbjni-6.29.4.1
scala-collection-compat_2.13-2.6.0
scala-library-2.13.8
scala-logging_2.13-3.9.4
scala-reflect-2.13.8
scala-java8-compat_2.13-1.0.2
snappy-java-1.1.8.4
zookeeper-3.6.3
zookeeper-jute-3.6.3
Expand Down Expand Up @@ -300,18 +303,18 @@ MIT License

argparse4j-0.7.0, see: licenses/argparse-MIT
jopt-simple-5.0.4, see: licenses/jopt-simple-MIT
slf4j-api-1.7.30, see: licenses/slf4j-MIT
slf4j-log4j12-1.7.30, see: licenses/slf4j-MIT
slf4j-api-1.7.36, see: licenses/slf4j-MIT
slf4j-reload4j-1.7.36, see: licenses/slf4j-MIT

---------------------------------------
BSD 2-Clause

zstd-jni-1.5.0-4 see: licenses/zstd-jni-BSD-2-clause
zstd-jni-1.5.2-1 see: licenses/zstd-jni-BSD-2-clause

---------------------------------------
BSD 3-Clause

jline-3.12.1, see: licenses/jline-BSD-3-clause
jline-3.21.0, see: licenses/jline-BSD-3-clause
paranamer-2.8, see: licenses/paranamer-BSD-3-clause

---------------------------------------
Expand Down
19 changes: 14 additions & 5 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,16 @@ Follow instructions in https://kafka.apache.org/quickstart
./gradlew integrationTest

### Force re-running tests without code change ###
./gradlew cleanTest test
./gradlew cleanTest unitTest
./gradlew cleanTest integrationTest
./gradlew -Prerun-tests test
./gradlew -Prerun-tests unitTest
./gradlew -Prerun-tests integrationTest

### Running a particular unit/integration test ###
./gradlew clients:test --tests RequestResponseTest

### Repeatedly running a particular unit/integration test ###
I=0; while ./gradlew clients:test -Prerun-tests --tests RequestResponseTest --fail-fast; do (( I=$I+1 )); echo "Completed run: $I"; sleep 1; done

### Running a particular test method within a unit/integration test ###
./gradlew core:test --tests kafka.api.ProducerFailureHandlingTest.testCannotSendToInternalTopic
./gradlew clients:test --tests org.apache.kafka.clients.MetadataTest.testTimeToNextUpdate
Expand Down Expand Up @@ -180,15 +183,21 @@ Please note for this to work you should create/update user maven settings (typic
...


### Installing the jars to the local Maven repository ###
The recommended command is:
### Installing ALL the jars to the local Maven repository ###
The recommended command to build for both Scala 2.12 and 2.13 is:

./gradlewAll publishToMavenLocal

For backwards compatibility, the following also works:

./gradlewAll install

### Installing specific projects to the local Maven repository ###

./gradlew -PskipSigning :streams:publishToMavenLocal

If needed, you can specify the Scala version with `-PscalaVersion=2.13`.

### Building the test jar ###
./gradlew testJar

Expand Down
18 changes: 18 additions & 0 deletions Vagrantfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ ec2_subnet_id = nil
# Only override this by setting it to false if you're running in a VPC and you
# are running Vagrant from within that VPC as well.
ec2_associate_public_ip = nil
ec2_iam_instance_profile_name = nil

ebs_volume_type = 'gp3'

jdk_major = '8'
jdk_full = '8u202-linux-x64'
Expand All @@ -60,6 +63,18 @@ if File.exists?(local_config_file) then
eval(File.read(local_config_file), binding, "Vagrantfile.local")
end

# override any instance type set by Vagrantfile.local or above via an environment variable
if ENV['INSTANCE_TYPE'] then
ec2_instance_type = ENV['INSTANCE_TYPE']
end

# choose size based on overridden size
if ec2_instance_type.start_with?("m3") then
ebs_volume_size = 20
else
ebs_volume_size = 40
end

# TODO(ksweeney): RAM requirements are not empirical and can probably be significantly lowered.
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.hostmanager.enabled = enable_hostmanager
Expand Down Expand Up @@ -118,9 +133,11 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
aws.region = ec2_region
aws.availability_zone = ec2_az
aws.instance_type = ec2_instance_type

aws.ami = ec2_ami
aws.security_groups = ec2_security_groups
aws.subnet_id = ec2_subnet_id
aws.block_device_mapping = [{ 'DeviceName' => '/dev/sda1', 'Ebs.VolumeType' => ebs_volume_type, 'Ebs.VolumeSize' => ebs_volume_size }]
# If a subnet is specified, default to turning on a public IP unless the
# user explicitly specifies the option. Without a public IP, Vagrant won't
# be able to SSH into the hosts unless Vagrant is also running in the VPC.
Expand All @@ -133,6 +150,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
region.spot_instance = ec2_spot_instance
region.spot_max_price = ec2_spot_max_price
end
aws.iam_instance_profile_name = ec2_iam_instance_profile_name

# Exclude some directories that can grow very large from syncing
override.vm.synced_folder ".", "/vagrant", type: "rsync", rsync__exclude: ['.git', 'core/data/', 'logs/', 'tests/results/', 'results/']
Expand Down
6 changes: 3 additions & 3 deletions bin/kafka-run-class.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ if [ -z "$INCLUDE_TEST_JARS" ]; then
fi

# Exclude jars not necessary for running commands.
regex="(-(test|test-sources|src|scaladoc|javadoc)\.jar|jar.asc)$"
regex="(-(test|test-sources|src|scaladoc|javadoc)\.jar|jar.asc|connect-file.*\.jar)$"
should_include_file() {
if [ "$INCLUDE_TEST_JARS" = true ]; then
return 0
Expand All @@ -48,7 +48,7 @@ should_include_file() {
base_dir=$(dirname $0)/..

if [ -z "$SCALA_VERSION" ]; then
SCALA_VERSION=2.13.6
SCALA_VERSION=2.13.8
if [[ -f "$base_dir/gradle.properties" ]]; then
SCALA_VERSION=`grep "^scalaVersion=" "$base_dir/gradle.properties" | cut -d= -f 2`
fi
Expand Down Expand Up @@ -171,7 +171,7 @@ do
CLASSPATH="$CLASSPATH:$dir/*"
done

for cc_pkg in "api" "transforms" "runtime" "file" "mirror" "mirror-client" "json" "tools" "basic-auth-extension"
for cc_pkg in "api" "transforms" "runtime" "mirror" "mirror-client" "json" "tools" "basic-auth-extension"
do
for file in "$base_dir"/connect/${cc_pkg}/build/libs/connect-${cc_pkg}*.jar;
do
Expand Down
2 changes: 1 addition & 1 deletion bin/windows/kafka-run-class.bat
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ set BASE_DIR=%CD%
popd

IF ["%SCALA_VERSION%"] EQU [""] (
set SCALA_VERSION=2.13.6
set SCALA_VERSION=2.13.8
)

IF ["%SCALA_BINARY_VERSION%"] EQU [""] (
Expand Down
17 changes: 17 additions & 0 deletions bin/windows/kafka-storage.bat
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
@echo off
rem Licensed to the Apache Software Foundation (ASF) under one or more
rem contributor license agreements. See the NOTICE file distributed with
rem this work for additional information regarding copyright ownership.
rem The ASF licenses this file to You under the Apache License, Version 2.0
rem (the "License"); you may not use this file except in compliance with
rem the License. You may obtain a copy of the License at
rem
rem http://www.apache.org/licenses/LICENSE-2.0
rem
rem Unless required by applicable law or agreed to in writing, software
rem distributed under the License is distributed on an "AS IS" BASIS,
rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
rem See the License for the specific language governing permissions and
rem limitations under the License.

"%~dp0kafka-run-class.bat" kafka.tools.StorageTool %*
Loading

0 comments on commit 9a644f1

Please sign in to comment.