Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix: Some issues with connection #1900

Merged
merged 4 commits into from
Aug 21, 2020
Merged

Fix: Some issues with connection #1900

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
eb74031
Merge pull request #32 from edp963/dev-0.3
xxxllluuu Jul 29, 2020
cb8700b
https://github.com/edp963/davinci/issues/1898 1. fix some issues
xxxllluuu Aug 21, 2020
8d2bf6c
Merge pull request #33 from edp963/dev-0.3
xxxllluuu Aug 21, 2020
f2f5298
Merge remote-tracking branch 'origin/dev-0.3' into dev-0.3
xxxllluuu Aug 21, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 28 additions & 8 deletions config/application.yml.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,14 +40,22 @@ jwtToken:

##your datasouce config
source:
initial-size: 2
initial-size: 1
min-idle: 1
max-wait: 6000
max-wait: 30000
max-active: 10
break-after-acquire-failure: true
connection-error-retry-attempts: 0
query-timeout: 600000
validationQueryTimeout: 30000
connection-error-retry-attempts: 1
time-between-eviction-runs-millis: 2000
min-evictable-idle-time-millis: 600000
max-evictable-idle-time-millis: 900000
test-while-idle: true
test-on-borrow: false
test-on-return: false
validation-query: select 1
validation-query-timeout: 10
keep-alive: false
filters: stat

enable-query-log: false
result-limit: 1000000
Expand All @@ -69,10 +77,22 @@ spring:
username: root
password: root
driver-class-name: com.mysql.jdbc.Driver
initial-size: 2
initial-size: 1
min-idle: 1
max-wait: 60000
max-wait: 30000
max-active: 10
break-after-acquire-failure: true
connection-error-retry-attempts: 1
time-between-eviction-runs-millis: 2000
min-evictable-idle-time-millis: 600000
max-evictable-idle-time-millis: 900000
test-while-idle: true
test-on-borrow: false
test-on-return: false
validation-query: select 1
validation-query-timeout: 10
keep-alive: false
filters: stat

## redis config
## please choose either of the two ways
Expand Down Expand Up @@ -156,7 +176,7 @@ screenshot:
timeout_second: 600
phantomjs_path: $your_phantomjs_path$
chromedriver_path: $your_chromedriver_path$
remote_webdriver_url:$your_remote_webdriver_url$
remote_webdriver_url: $your_remote_webdriver_url$

data-auth-center:
channels:
Expand Down
131 changes: 84 additions & 47 deletions server/src/main/java/edp/core/common/jdbc/JdbcDataSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,25 +19,14 @@

package edp.core.common.jdbc;

import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_CONNECTIONPROPERTIES;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_INITIALSIZE;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_MAXACTIVE;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_MAXWAIT;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_MINEVICTABLEIDLETIMEMILLIS;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_MINIDLE;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_PASSWORD;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_TESTONBORROW;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_TESTONRETURN;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_TESTWHILEIDLE;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_TIMEBETWEENEVICTIONRUNSMILLIS;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_URL;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_USERNAME;
import static com.alibaba.druid.pool.DruidDataSourceFactory.*;
import static edp.core.consts.Consts.JDBC_DATASOURCE_DEFAULT_VERSION;

import java.io.File;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
Expand Down Expand Up @@ -79,17 +68,27 @@ public DruidDataSource getDataSource(JdbcSourceInfo jdbcSourceInfo) throws Sourc
if (druidDataSource != null && !druidDataSource.isClosed()) {
return druidDataSource;
}

Lock lock = getDataSourceLock(key);

try {
if (!lock.tryLock(5L, TimeUnit.SECONDS)) {
throw new SourceException("Unable to get driver instance for jdbcUrl: " + jdbcUrl);
}
}
catch (InterruptedException e) {
throw new SourceException("Unable to get driver instance for jdbcUrl: " + jdbcUrl);
}

try {
if (!lock.tryLock(60L, TimeUnit.SECONDS)) {
druidDataSource = dataSourceMap.get(key);
if (druidDataSource != null && !druidDataSource.isClosed()) {
return druidDataSource;
}
throw new SourceException("Unable to get datasource for jdbcUrl: " + jdbcUrl);
}
}
catch (InterruptedException e) {
throw new SourceException("Unable to get datasource for jdbcUrl: " + jdbcUrl);
}

druidDataSource = dataSourceMap.get(key);
if (druidDataSource != null && !druidDataSource.isClosed()) {
lock.unlock();
return druidDataSource;
}

druidDataSource = dataSourceMap.get(key);
if (druidDataSource != null && !druidDataSource.isClosed()) {
Expand All @@ -112,9 +111,15 @@ public DruidDataSource getDataSource(JdbcSourceInfo jdbcSourceInfo) throws Sourc
properties.setProperty(PROP_MAXWAIT, String.valueOf(maxWait));
properties.setProperty(PROP_TIMEBETWEENEVICTIONRUNSMILLIS, String.valueOf(timeBetweenEvictionRunsMillis));
properties.setProperty(PROP_MINEVICTABLEIDLETIMEMILLIS, String.valueOf(minEvictableIdleTimeMillis));
properties.setProperty(PROP_TESTWHILEIDLE, String.valueOf(false));
properties.setProperty(PROP_MAXOPENPREPAREDSTATEMENTS, String.valueOf(maxEvictableIdleTimeMillis));
properties.setProperty(PROP_TESTWHILEIDLE, String.valueOf(testWhileIdle));
properties.setProperty(PROP_TESTONBORROW, String.valueOf(testOnBorrow));
properties.setProperty(PROP_TESTONRETURN, String.valueOf(testOnReturn));
properties.setProperty(PROP_VALIDATIONQUERY, validationQuery);
properties.setProperty(PROP_VALIDATIONQUERY_TIMEOUT, String.valueOf(validationQueryTimeout));
properties.setProperty(PROP_REMOVEABANDONED, "true");
properties.setProperty(PROP_REMOVEABANDONEDTIMEOUT, "3900");
properties.setProperty(PROP_LOGABANDONED, "true");
properties.put(PROP_CONNECTIONPROPERTIES, "client.transport.ignore_cluster_name=true");

if (!CollectionUtils.isEmpty(jdbcSourceInfo.getProperties())) {
Expand All @@ -138,56 +143,69 @@ public DruidDataSource getDataSource(JdbcSourceInfo jdbcSourceInfo) throws Sourc
@Autowired
private ESDataSource esDataSource;

@Value("${spring.datasource.type}")
protected String type;

@Value("${source.max-active:10}")
@Value("${source.max-active:8}")
@Getter
protected int maxActive;

@Value("${source.initial-size:1}")
@Value("${source.initial-size:0}")
@Getter
protected int initialSize;

@Value("${source.min-idle:3}")
@Value("${source.min-idle:1}")
@Getter
protected int minIdle;

@Value("${source.max-wait:30000}")
@Value("${source.max-wait:60000}")
@Getter
protected long maxWait;

@Value("${spring.datasource.time-between-eviction-runs-millis}")
@Value("${source.time-between-eviction-runs-millis}")
@Getter
protected long timeBetweenEvictionRunsMillis;

@Value("${spring.datasource.min-evictable-idle-time-millis}")
@Value("${source.min-evictable-idle-time-millis}")
@Getter
protected long minEvictableIdleTimeMillis;

@Value("${spring.datasource.test-while-idle}")
@Value("${source.max-evictable-idle-time-millis}")
@Getter
protected long maxEvictableIdleTimeMillis;

@Value("${source.test-while-idle}")
@Getter
protected boolean testWhileIdle;

@Value("${spring.datasource.test-on-borrow}")
@Value("${source.test-on-borrow}")
@Getter
protected boolean testOnBorrow;

@Value("${spring.datasource.test-on-return}")
@Value("${source.test-on-return}")
@Getter
protected boolean testOnReturn;

@Value("${source.break-after-acquire-failure:true}")
@Getter
protected boolean breakAfterAcquireFailure;

@Value("${source.connection-error-retry-attempts:0}")
@Value("${source.connection-error-retry-attempts:1}")
@Getter
protected int connectionErrorRetryAttempts;

@Value("${source.query-timeout:600000}")
@Value("${source.keep-alive:false}")
@Getter
protected int queryTimeout;
protected boolean keepAlive;

@Value("${source.validation-query-timeout:5}")
@Getter
protected int validationQueryTimeout;

@Value("${source.validation-query}")
@Getter
protected String validationQuery;

@Value("${source.filters}")
@Getter
protected String filters;

private static volatile Map<String, DruidDataSource> dataSourceMap = new ConcurrentHashMap<>();
private static volatile Map<String, Lock> dataSourceLockMap = new ConcurrentHashMap<>();
Expand Down Expand Up @@ -223,7 +241,7 @@ public void removeDatasource(JdbcSourceInfo jdbcSourceInfo) {
String key = getDataSourceKey(jdbcSourceInfo);

Lock lock = getDataSourceLock(key);

if (!lock.tryLock()) {
return;
}
Expand Down Expand Up @@ -262,16 +280,21 @@ public DruidDataSource getDataSource(JdbcSourceInfo jdbcSourceInfo) throws Sourc
Lock lock = getDataSourceLock(key);

try {
if (!lock.tryLock(5L, TimeUnit.SECONDS)) {
throw new SourceException("Unable to get driver instance for jdbcUrl: " + jdbcUrl);
if (!lock.tryLock(30L, TimeUnit.SECONDS)) {
druidDataSource = dataSourceMap.get(key);
if (druidDataSource != null && !druidDataSource.isClosed()) {
return druidDataSource;
}
throw new SourceException("Unable to get datasource for jdbcUrl: " + jdbcUrl);
}
}
catch (InterruptedException e) {
throw new SourceException("Unable to get driver instance for jdbcUrl: " + jdbcUrl);
throw new SourceException("Unable to get datasource for jdbcUrl: " + jdbcUrl);
}

druidDataSource = dataSourceMap.get(key);
if (druidDataSource != null && !druidDataSource.isClosed()) {
lock.unlock();
return druidDataSource;
}

Expand Down Expand Up @@ -310,24 +333,38 @@ public DruidDataSource getDataSource(JdbcSourceInfo jdbcSourceInfo) throws Sourc
druidDataSource.setMaxWait(maxWait);
druidDataSource.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis);
druidDataSource.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis);
druidDataSource.setTestWhileIdle(false);
druidDataSource.setMaxEvictableIdleTimeMillis(maxEvictableIdleTimeMillis);
druidDataSource.setTestWhileIdle(testWhileIdle);
druidDataSource.setTestOnBorrow(testOnBorrow);
druidDataSource.setTestOnReturn(testOnReturn);
druidDataSource.setConnectionErrorRetryAttempts(connectionErrorRetryAttempts);
druidDataSource.setBreakAfterAcquireFailure(breakAfterAcquireFailure);

String driverName=druidDataSource.getDriverClassName();
if(driverName.indexOf("sqlserver")!=-1) {
druidDataSource.setKeepAlive(keepAlive);
druidDataSource.setValidationQueryTimeout(validationQueryTimeout);
druidDataSource.setValidationQuery(validationQuery);
druidDataSource.setRemoveAbandoned(true);
druidDataSource.setRemoveAbandonedTimeout(3600 + 5 * 60);
druidDataSource.setLogAbandoned(true);

// default validation query
String driverName = druidDataSource.getDriverClassName();
if (driverName.indexOf("sqlserver") != -1 || driverName.indexOf("mysql") != -1
|| driverName.indexOf("h2") != -1 || driverName.indexOf("moonbox") != -1) {
druidDataSource.setValidationQuery("select 1");
}

if (driverName.indexOf("oracle") != -1) {
druidDataSource.setValidationQuery("select 1 from dual");
}

if (!CollectionUtils.isEmpty(jdbcSourceInfo.getProperties())) {
Properties properties = new Properties();
jdbcSourceInfo.getProperties().forEach(dict -> properties.setProperty(dict.getKey(), dict.getValue()));
druidDataSource.setConnectProperties(properties);
}

try {
druidDataSource.setFilters(filters);
druidDataSource.init();
} catch (Exception e) {
log.error("Exception during pool initialization", e);
Expand Down
8 changes: 8 additions & 0 deletions server/src/main/java/edp/core/config/DruidConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ public class DruidConfig {
@Value("${spring.datasource.min-evictable-idle-time-millis}")
private int minEvictableIdleTimeMillis;

@Value("${spring.datasource.max-evictable-idle-time-millis}")
private int maxEvictableIdleTimeMillis;

@Value("${spring.datasource.test-while-idle}")
private boolean testWhileIdle;

Expand All @@ -93,6 +96,9 @@ public class DruidConfig {
@Value("${spring.datasource.validation-query}")
private String validationQuery;

@Value("${spring.datasource.validation-query-timeout}")
private int validationQueryTime;

/**
* druid监控
*
Expand Down Expand Up @@ -139,12 +145,14 @@ public DruidDataSource druidDataSource() {
druidDataSource.setMaxWait(maxWait);
druidDataSource.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis);
druidDataSource.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis);
druidDataSource.setMaxEvictableIdleTimeMillis(maxEvictableIdleTimeMillis);
druidDataSource.setTestWhileIdle(testWhileIdle);
druidDataSource.setTestOnBorrow(testOnBorrow);
druidDataSource.setTestOnReturn(testOnReturn);
druidDataSource.setBreakAfterAcquireFailure(breakAfterAcquireFailure);
druidDataSource.setConnectionErrorRetryAttempts(connectionErrorRetryAttempts);
druidDataSource.setValidationQuery(validationQuery);
druidDataSource.setValidationQueryTimeout(validationQueryTime);

try {
druidDataSource.setFilters(filters);
Expand Down
26 changes: 26 additions & 0 deletions server/src/main/java/edp/core/utils/DateUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -314,4 +314,30 @@ public static java.sql.Date toSqlDate(Long timeLongInMicros) {
return toSqlDate(new Date(timeLongInMicros));
}

public static Date dateFormat(String date, String dateFormat) {
if (date == null) {
return null;
}
SimpleDateFormat format = new SimpleDateFormat(dateFormat);
try {
return format.parse(date);
} catch (Exception ex) {

}
return null;
}

public static String dateFormat(Date date, String dateFormat) {
if (date == null) {
return "";
}
SimpleDateFormat format = new SimpleDateFormat(dateFormat);
try {
return format.format(date);
} catch (Exception ex) {

}
return "";
}

}
10 changes: 10 additions & 0 deletions server/src/main/java/edp/core/utils/FileUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.FileCopyUtils;
import org.springframework.web.multipart.MultipartFile;

import javax.imageio.ImageIO;
Expand Down Expand Up @@ -367,4 +368,13 @@ public static void closeCloseable(Closeable c) {
}
}
}

public static int copy(File in, File out) {
try {
return FileCopyUtils.copy(in, out);
} catch (IOException e) {
e.printStackTrace();
}
return -1;
}
}
Loading