Skip to content

Commit

Permalink
DB and RabbitMQ integration tests and PostgreSQL testing/bug fixing (#…
Browse files Browse the repository at this point in the history
…1047)

* Initial matrixSigned-off-by: Kai Zimmermann <kai.zimmermann@microsoft.com>

* License header

* MySQL DB testSigned-off-by: Kai Zimmermann <kai.zimmermann@microsoft.com>

* Create matrix for DBsSigned-off-by: Kai Zimmermann <kai.zimmermann@microsoft.com>

* RabbitMQ and H2Signed-off-by: Kai Zimmermann <kai.zimmermann@microsoft.com>

* MySQL 8.0Signed-off-by: Kai Zimmermann <kai.zimmermann@microsoft.com>

* Postgresql test supportSigned-off-by: Kai Zimmermann <kai.zimmermann@microsoft.com>

* Postgresql test supportSigned-off-by: Kai Zimmermann <kai.zimmermann@microsoft.com>

* Fix DB issues post and mssql

Signed-off-by: Kai Zimmermann <kai.zimmermann@microsoft.com>

* Wait MSSQL

Signed-off-by: Kai Zimmermann <kai.zimmermann@microsoft.com>

* Fix postgresql tests.

Signed-off-by: Kai Zimmermann <kai.zimmermann@microsoft.com>

* MSSQL startup fix.Signed-off-by: Kai Zimmermann <kai.zimmermann@microsoft.com>

* Fix syntax error

Signed-off-by: Kai Zimmermann <kai.zimmermann@microsoft.com>

* Further fix postgres tests.Signed-off-by: Kai Zimmermann <kai.zimmermann@microsoft.com>

* Revert unnecessary changes.

Signed-off-by: Kai Zimmermann <kai.zimmermann@microsoft.com>

* Add SonarCloud Signed-off-by: Kai Zimmermann <kai.zimmermann@microsoft.com>

* Simplify devcontainer. Test JDK 15Signed-off-by: Kai Zimmermann <kai.zimmermann@microsoft.com>
  • Loading branch information
kaizimmerm authored Jan 14, 2021
1 parent 4ea5d76 commit e9f11d2
Show file tree
Hide file tree
Showing 27 changed files with 557 additions and 114 deletions.
119 changes: 119 additions & 0 deletions .azure-pipelines/integration-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
# Build hawkBit and run tests with Apache Maven.
# Runs a matrix of various DB,JDK,RabbitMQ versions that are supported by hawkBit
#
# Requires the SonarCloud plugin and connection setup
# https://kaizimmerm.com/post/azure-pipelines-for-the-oss-developer/#analyze-the-code-with-sonarcloud
#
# Add steps that analyze code, save build artifacts, deploy, and more:
# https://docs.microsoft.com/en-us/azure/devops/pipelines/ecosystems/java
# https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema
# https://docs.microsoft.com/en-us/azure/devops/pipelines/release/caching?view=azure-devops#maven

pool:
vmImage: "ubuntu-18.04"

# Default values for the hawbit build. Can be overriden in the variable group 'hawkbit'
# see https://docs.microsoft.com/en-us/azure/devops/pipelines/library/variable-groups
variables:
# Defines defaults
- name: sonarCloudConnection
value: 'hawkBitSonar'
- name: sonarCloudOrganization
value: 'hawkbit'
- name: sonarProjectKey
value: 'org.eclipse:hawkbit'
# Override defaults
- group: hawkbit

jobs:
- job: JDK_11
displayName: Verify with JDK-11 and SonarCloud analysis
steps:
- template: rabbitmq-template.yml
- template: maven-template.yml
parameters:
mavenGoals: "verify -Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=$(sonarCloudOrganization) -Dsonar.projectKey=$(sonarProjectKey)"
jdkVersionOption: "1.11"
sonarQubeRunAnalysis: true
sonarCloudConnection: $(sonarCloudConnection)
sonarCloudOrganization: $(sonarCloudOrganization)
- job: JDK_8
displayName: Build with JDK-8 (hawkBit default)
steps:
- template: maven-template.yml
parameters:
mavenGoals: "install license:check"
- job:
dependsOn: JDK_8
condition: succeeded()
displayName: RABBIT
strategy:
matrix:
3.6:
rabbitmqVersion: "3.6"
3.7:
rabbitmqVersion: "3.7"
3.8:
rabbitmqVersion: "3.8"
steps:
- template: rabbitmq-template.yml
parameters:
rabbitmqVersion: $(rabbitmqVersion)
- template: maven-template.yml
parameters:
mavenGoals: "verify"
- job:
dependsOn: JDK_8
condition: succeeded()
displayName: MYSQL
strategy:
matrix:
5.6:
dbVersion: "5.6"
5.7:
dbVersion: "5.7"
steps:
- template: rabbitmq-template.yml
- script: "docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=8236472364 -e MYSQL_DATABASE=hawkbit -d mysql:$(dbVersion)"
displayName: "Setup MYSQL Database docker instance"
- template: maven-template.yml
parameters:
mavenGoals: "verify -Dspring.jpa.database=MYSQL -Dspring.datasource.driverClassName=org.mariadb.jdbc.Driver -Dspring.datasource.url=jdbc:mysql://localhost:3306/hawkbit -Dspring.datasource.username=root -Dspring.datasource.password=8236472364"
- job:
dependsOn: JDK_8
condition: succeeded()
displayName: MSSQL
strategy:
matrix:
2017:
dbVersion: "2017-latest"
2019:
dbVersion: "2019-latest"
steps:
- template: rabbitmq-template.yml
- script: |
docker run --name mssql -p 1433:1433 -e ACCEPT_EULA=Y -e SA_PASSWORD=1234567890.Ab -d mcr.microsoft.com/mssql/server:$(dbVersion)
until [ "`/usr/bin/docker inspect -f {{.State.Running}} mssql`" == "true" ]; do sleep 1; done
sleep 120
until docker exec mssql /opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P "1234567890.Ab" -Q "CREATE DATABASE hawkbit"; do sleep 1; done
displayName: "Setup MSSQL Database docker instance"
- template: maven-template.yml
parameters:
mavenGoals: "verify -Dspring.jpa.database=SQL_SERVER -Dspring.datasource.url=jdbc:sqlserver://localhost:1433;database=hawkbit -Dspring.datasource.username=SA -Dspring.datasource.password=1234567890.Ab -Dspring.datasource.driverClassName=com.microsoft.sqlserver.jdbc.SQLServerDriver"
- job:
dependsOn: JDK_8
condition: succeeded()
displayName: POSTGRESQL
strategy:
matrix:
12:
dbVersion: "12"
13:
dbVersion: "13"
steps:
- template: rabbitmq-template.yml
- script: "docker run --name postgres -p 5432:5432 -e POSTGRES_PASSWORD=1234567890 -e POSTGRES_DB=hawkbit -d postgres:$(dbVersion)"
displayName: "Setup POSTGRESQL Database docker instance"
- template: maven-template.yml
parameters:
mavenGoals: "verify -Dspring.jpa.database=POSTGRESQL -Dspring.datasource.url=jdbc:postgresql://localhost:5432/hawkbit?currentSchema=hawkbit -Dspring.datasource.username=postgres -Dspring.datasource.password=1234567890 -Dspring.datasource.driverClassName=org.postgresql.Driver"
50 changes: 50 additions & 0 deletions .azure-pipelines/maven-template.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
parameters:
- name: mavenGoals
displayName: Maven Goal
type: string
- name: jdkVersionOption
displayName: JDK Version
type: string
default: "1.8"
- name: sonarQubeRunAnalysis
displayName: Enable SonarQube analysis
type: boolean
default: false
- name: sonarCloudConnection
displayName: Optional SonarCloud connection
type: string
default: ''
- name: sonarCloudOrganization
displayName: Optional SonarCloud organization
type: string
default: ''
- name: mavenCacheFolder
displayName: Maven Cache Folder
type: string
default: $(Pipeline.Workspace)/.m2/repository

steps:
- task: SonarCloudPrepare@1
condition: eq('${{ parameters.sonarQubeRunAnalysis }}', true)
displayName: 'Prepare SonarCloud analysis configuration'
inputs:
SonarCloud: ${{ parameters.sonarCloudConnection }}
organization: ${{ parameters.sonarCloudOrganization }}
scannerMode: Other
- task: Cache@2
inputs:
key: 'maven | "$(Agent.OS)" | "${{ parameters.jdkVersionOption }}" | **/pom.xml'
path: ${{ parameters.mavenCacheFolder }}
displayName: Cache Maven local repo
- task: Maven@3
inputs:
mavenPomFile: "pom.xml"
mavenOptions: "-Xmx3072m"
javaHomeOption: "JDKVersion"
jdkVersionOption: ${{ parameters.jdkVersionOption }}
jdkArchitectureOption: "x64"
publishJUnitResults: true
sonarQubeRunAnalysis: ${{ parameters.sonarQubeRunAnalysis }}
sqMavenPluginVersionChoice: 'latest'
testResultsFiles: "**/surefire-reports/TEST-*.xml"
goals: "${{ parameters.mavenGoals }} -Dmaven.repo.local=${{ parameters.mavenCacheFolder }}"
9 changes: 9 additions & 0 deletions .azure-pipelines/rabbitmq-template.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
parameters:
- name: rabbitmqVersion
displayName: RabbitMQ Version
type: string
default: "3.8"

steps:
- script: docker run -d --name rabbit -p 15672:15672 -p 5672:5672 -e RABBITMQ_DEFAULT_VHOST=/ rabbitmq:${{ parameters.rabbitmqVersion }}-management
displayName: "Setup RabbitMQ docker instance"
9 changes: 9 additions & 0 deletions .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# [Choice] Java version: 8. 11, 15
ARG VARIANT=8
FROM mcr.microsoft.com/vscode/devcontainers/java:${VARIANT}

RUN su vscode -c "source /usr/local/sdkman/bin/sdkman-init.sh && sdk install maven \"${MAVEN_VERSION}\""

# [Optional] Uncomment this section to install additional OS packages.
# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
# && apt-get -y install --no-install-recommends <your-package-list-here>
35 changes: 35 additions & 0 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
{
"name": "Java",
"build": {
"dockerfile": "Dockerfile",
"args": {
// Update the VARIANT arg to pick a Java version: 8, 11, 15
"VARIANT": "8"
}
},
// Set *default* container specific settings.json values on container create.
"settings": {
"terminal.integrated.shell.linux": "/bin/bash",
"java.home": "/docker-java-home",
"java.format.settings.url": "https://raw.githubusercontent.com/eclipse/hawkbit/master/eclipse_codeformatter.xml",
"maven.executable.path": "/usr/local/sdkman/candidates/maven/current/bin/mvn"
},
// Add the IDs of extensions you want installed when the container is created.
"extensions": [
"vscjava.vscode-java-pack",
"DavidAnson.vscode-markdownlint",
"ms-azuretools.vscode-docker",
"ms-azure-devops.azure-pipelines",
"sonarsource.sonarlint-vscode",
"pivotal.vscode-spring-boot",
"vscjava.vscode-spring-boot-dashboard"
],
// Use 'forwardPorts' to make a list of ports inside the container available locally.
"forwardPorts": [
8080
],
// Use 'postCreateCommand' to run commands after the container is created.
// "postCreateCommand": "java -version",
// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
"remoteUser": "vscode"
}
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,20 +48,20 @@ There are clients outside of the Eclipse IoT eco system as well, e.g.:

# Runtime dependencies and support

## Java Runtime Environment: 1.8
## Java Runtime Environment: 1.8,11

## SQL database

| Database | H2 | MySQL/MariaDB | MS SQL Server | PostgreSQL | IBM DB2 |
| --------------------------------- | :------------------------------------------------: | :-----------------------------------------------------------------------: | :-------------------------------------------------------: | :----------------------------------------------------------------: | :----------------: |
| DDLs maintained by project | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
| Test dependencies defined | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | |
| Versions tested | 1.4 | MySQL 5.6/5.7, AWS Aurora | MS SQL Server 2017 | PostgreSQL 12.1 | DB2 Server v11.1 |
| Versions tested | 1.4 | MySQL 5.6/5.7, AWS Aurora | MS SQL Server 2017/2019 | PostgreSQL 12/13 | DB2 Server v11.1 |
| Docker image with driver provided | :white_check_mark: | :white_check_mark: (Tag: "-mysql") | :white_check_mark: | :white_check_mark: | |
| JDBC driver | [H2 1.4](https://github.com/h2database/h2database) | [MariaDB Connector/J 2.0](https://github.com/MariaDB/mariadb-connector-j) | [MSSQL-JDBC 6.4](https://github.com/Microsoft/mssql-jdbc) | [PostgreSQL JDBC Driver 42.2.10](https://github.com/pgjdbc/pgjdbc) | |
| Status | Test, Dev | Production grade | Production grade | Test, Dev | Test, Dev |

## (Optional) RabbitMQ: 3.6,3.7
## (Optional) RabbitMQ: 3.6,3.7,3.8

# Getting Started

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,9 @@ public class JpaDeploymentManagement extends JpaActionManagement implements Depl
QUERY_DELETE_ACTIONS_BY_STATE_AND_LAST_MODIFIED = new EnumMap<>(Database.class);
QUERY_DELETE_ACTIONS_BY_STATE_AND_LAST_MODIFIED.put(Database.SQL_SERVER, "DELETE TOP (" + ACTION_PAGE_LIMIT
+ ") FROM sp_action WHERE tenant=#tenant AND status IN (%s) AND last_modified_at<#last_modified_at ");
QUERY_DELETE_ACTIONS_BY_STATE_AND_LAST_MODIFIED.put(Database.POSTGRESQL,
"DELETE FROM sp_action WHERE id IN (SELECT id FROM sp_action WHERE tenant=#tenant AND status IN (%s) AND last_modified_at<#last_modified_at LIMIT "
+ ACTION_PAGE_LIMIT + ")");
}

private final EntityManager entityManager;
Expand Down Expand Up @@ -209,8 +212,8 @@ private List<DistributionSetAssignmentResult> assignDistributionSets(final Strin
final Map<Long, List<TargetWithActionType>> assignmentsByDsIds = convertRequest(validatedRequests);

final List<DistributionSetAssignmentResult> results = assignmentsByDsIds.entrySet().stream()
.map(entry -> assignDistributionSetToTargetsWithRetry(initiatedBy, entry.getKey(), entry.getValue(), actionMessage,
strategy))
.map(entry -> assignDistributionSetToTargetsWithRetry(initiatedBy, entry.getKey(), entry.getValue(),
actionMessage, strategy))
.collect(Collectors.toList());
strategy.sendDeploymentEvents(results);
return results;
Expand Down Expand Up @@ -239,8 +242,8 @@ private static void checkIfRequiresMultiAssignment(final Collection<DeploymentRe
}
}

private DistributionSetAssignmentResult assignDistributionSetToTargetsWithRetry(final String initiatedBy, final Long dsID,
final Collection<TargetWithActionType> targetsWithActionType, final String actionMessage,
private DistributionSetAssignmentResult assignDistributionSetToTargetsWithRetry(final String initiatedBy,
final Long dsID, final Collection<TargetWithActionType> targetsWithActionType, final String actionMessage,
final AbstractDsAssignmentStrategy assignmentStrategy) {
final RetryCallback<DistributionSetAssignmentResult, ConcurrencyFailureException> retryCallback = retryContext -> assignDistributionSetToTargets(
initiatedBy, dsID, targetsWithActionType, actionMessage, assignmentStrategy);
Expand Down Expand Up @@ -427,9 +430,7 @@ private List<JpaAction> createActions(final String initiatedBy,

return targetsWithActionType.stream()
.map(twt -> assignmentStrategy.createTargetAction(initiatedBy, twt, targets, set))
.filter(Objects::nonNull)
.map(actionRepository::save)
.collect(Collectors.toList());
.filter(Objects::nonNull).map(actionRepository::save).collect(Collectors.toList());
}

private void createActionsStatus(final Collection<JpaAction> actions,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,15 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.orm.jpa.JpaProperties;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.cache.interceptor.KeyGenerator;
import org.springframework.dao.ConcurrencyFailureException;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.orm.jpa.vendor.Database;
import org.springframework.retry.annotation.Backoff;
import org.springframework.retry.annotation.Retryable;
import org.springframework.transaction.PlatformTransactionManager;
Expand Down Expand Up @@ -123,11 +125,15 @@ public class JpaSystemManagement implements CurrentTenantCacheKeyGenerator, Syst
@Autowired
private ArtifactRepository artifactRepository;

@Autowired
private JpaProperties properties;

@Override
public SystemUsageReport getSystemUsageStatistics() {

final Number count = (Number) entityManager.createNativeQuery(
"select SUM(file_size) from sp_artifact a INNER JOIN sp_base_software_module sm ON a.software_module = sm.id WHERE sm.deleted = 0")
"select SUM(file_size) from sp_artifact a INNER JOIN sp_base_software_module sm ON a.software_module = sm.id WHERE sm.deleted = "
+ (isPostgreSql(properties) ? "false" : "0"))
.getSingleResult();

long sumOfArtifacts = 0;
Expand All @@ -141,7 +147,8 @@ public SystemUsageReport getSystemUsageStatistics() {
.getSingleResult()).longValue();

final long artifacts = ((Number) entityManager.createNativeQuery(
"SELECT COUNT(a.id) FROM sp_artifact a INNER JOIN sp_base_software_module sm ON a.software_module = sm.id WHERE sm.deleted = 0")
"SELECT COUNT(a.id) FROM sp_artifact a INNER JOIN sp_base_software_module sm ON a.software_module = sm.id WHERE sm.deleted = "
+ (isPostgreSql(properties) ? "false" : "0"))
.getSingleResult()).longValue();

final long actions = ((Number) entityManager.createNativeQuery("SELECT COUNT(id) FROM sp_action")
Expand All @@ -151,6 +158,10 @@ public SystemUsageReport getSystemUsageStatistics() {
tenantMetaDataRepository.count());
}

private static boolean isPostgreSql(final JpaProperties properties) {
return Database.POSTGRESQL.equals(properties.getDatabase());
}

@Override
public SystemUsageReportWithTenants getSystemUsageStatisticsWithTenants() {
final SystemUsageReportWithTenants result = (SystemUsageReportWithTenants) getSystemUsageStatistics();
Expand Down
Loading

0 comments on commit e9f11d2

Please sign in to comment.