Skip to content

Commit

Permalink
Various fixes
Browse files Browse the repository at this point in the history
- Common/R66: improve multithreading usage
- Compression: make Zlib codec usable for FTP (mode Z) (only using compatible clients, such as WaarpFtp4J based o FTP4J)
- Packaging: following issue #2 from fb clone on packaging using release profile
- Update dependencies (including issue #1 from fb clone)
- Try to fix and improve FTP concurrency (including issue waarp#99)
- Fix documentation
  • Loading branch information
fredericBregier committed Aug 18, 2021
1 parent 493f1cf commit cbf8466
Show file tree
Hide file tree
Showing 93 changed files with 2,261 additions and 803 deletions.
18 changes: 18 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,24 @@ cd Waarp-All
mvn -P jre11 package
```

If you want to build unofficial RPM/DEV/TGZ/ZIP and documentation, you can do as the following,
ensuring you have already cloned and install using `pip` the repo for Sphinx template for Waarp
`code.waarp.fr:2222/waarp/sphinx-template.git` with the following packages for Sphinx:

- sphinx
- sphinx-autobuild
- sphinxcontrib-httpdomain
- Possibly fix the current version 1.6 to 1.7
- `sphinxcontrib/httpdomain.py`
- line 766
- `+ app.add_domain(HTTPDomain)`
- sphinxcontrib-openapi
- sphinx.ext.todo

```sh
mvn -P jre11,release package
```

You can use a JDK 11 (or higher) with `jre11` profile, and a JDK 8 with `jre8` or `jre6` profiles.

`mvn -P jre11 package` also runs the full test suite, which takes quite some time (for more
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,8 @@ private void initialize(final DbModel dbModel, final String server,
} catch (final SQLException ex) {
setDisActive(true);
// handle any errors
logger.error(CANNOT_CREATE_CONNECTION);
logger.error(CANNOT_CREATE_CONNECTION + " while already having {}",
DbAdmin.getNbConnection());
error(ex);
if (getConn() != null) {
try {
Expand Down Expand Up @@ -223,7 +224,8 @@ public final void setAutoCommit(final boolean autoCommit)
getConn().setAutoCommit(autoCommit);
} catch (final SQLException e) {
// handle any errors
logger.error(CANNOT_CREATE_CONNECTION);
logger.error(CANNOT_CREATE_CONNECTION + " while already having {}",
DbAdmin.getNbConnection());
error(e);
if (getConn() != null) {
try {
Expand Down
19 changes: 13 additions & 6 deletions WaarpCommon/src/main/java/org/waarp/common/file/DataBlock.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import org.waarp.common.utility.WaarpNettyUtil;

/**
* Main object implementing Data Block whaveter the mode, type, structure used.
Expand Down Expand Up @@ -139,11 +140,11 @@ public final void setBlock(final ByteBuf block) {
this.block = new byte[byteCount];
offsetBuf = 0;
if (blockBuf != null) {
blockBuf.release();
WaarpNettyUtil.release(blockBuf);
blockBuf = null;
}
block.readBytes(this.block);
block.release();
WaarpNettyUtil.release(block);
}

/**
Expand All @@ -165,8 +166,14 @@ public final void setBlock(final byte[] block, final int size) {
if (isRESTART) {
this.block = null;
markers = new int[6];
for (int i = 0; i < 6; i++) {
markers[i] = block[i];
if (block == null) {
for (int i = 0; i < 6; i++) {
markers[i] = 0;
}
} else {
for (int i = 0; i < 6; i++) {
markers[i] = block[i];
}
}
byteCount = 6;
return;
Expand All @@ -178,7 +185,7 @@ public final void setBlock(final byte[] block, final int size) {
byteCount = size;
}
if (blockBuf != null) {
blockBuf.release();
WaarpNettyUtil.release(blockBuf);
blockBuf = null;
}
offsetBuf = 0;
Expand Down Expand Up @@ -337,7 +344,7 @@ public final void setMarkers(final int[] markers) {
*/
public final void clear() {
if (blockBuf != null) {
blockBuf.release();
WaarpNettyUtil.release(blockBuf);
blockBuf = null;
}
block = null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,7 @@ public final String getFile() throws CommandAbstractException {
}

@Override
public boolean closeFile() throws CommandAbstractException {
public synchronized boolean closeFile() throws CommandAbstractException {
if (fileInputStream != null) {
FileUtils.close(fileInputStream);
fileInputStream = null;
Expand All @@ -295,7 +295,8 @@ public boolean closeFile() throws CommandAbstractException {
}

@Override
public final boolean abortFile() throws CommandAbstractException {
public final synchronized boolean abortFile()
throws CommandAbstractException {
if (isInWriting() &&
((FilesystemBasedFileParameterImpl) getSession().getFileParameter()).deleteOnAbort) {
delete();
Expand All @@ -320,15 +321,15 @@ public long length() throws CommandAbstractException {
}

@Override
public final boolean isInReading() {
public final synchronized boolean isInReading() {
if (!isReady) {
return false;
}
return fileInputStream != null;
}

@Override
public final boolean isInWriting() {
public final synchronized boolean isInWriting() {
if (!isReady) {
return false;
}
Expand Down Expand Up @@ -473,7 +474,7 @@ public boolean renameTo(final String path) throws CommandAbstractException {
}

@Override
public final DataBlock readDataBlock()
public final synchronized DataBlock readDataBlock()
throws FileTransferException, FileEndOfTransferException {
if (isReady) {
return getByteBlock(getSession().getBlockSize());
Expand All @@ -482,7 +483,7 @@ public final DataBlock readDataBlock()
}

@Override
public final DataBlock readDataBlock(final byte[] bufferGiven)
public final synchronized DataBlock readDataBlock(final byte[] bufferGiven)
throws FileTransferException, FileEndOfTransferException {
if (isReady) {
return getByteBlock(bufferGiven);
Expand All @@ -491,7 +492,7 @@ public final DataBlock readDataBlock(final byte[] bufferGiven)
}

@Override
public final void writeDataBlock(final DataBlock dataBlock)
public final synchronized void writeDataBlock(final DataBlock dataBlock)
throws FileTransferException {
if (isReady) {
if (dataBlock.isEOF()) {
Expand All @@ -514,7 +515,7 @@ public final void writeDataBlock(final DataBlock dataBlock)
*
* @return the position
*/
public final long getPosition() {
public final synchronized long getPosition() {
return position;
}

Expand All @@ -526,7 +527,8 @@ public final long getPosition() {
* @throws IOException
*/
@Override
public final void setPosition(final long position) throws IOException {
public final synchronized void setPosition(final long position)
throws IOException {
if (this.position != position) {
this.position = position;
if (fileInputStream != null) {
Expand Down Expand Up @@ -556,8 +558,9 @@ public final void setPosition(final long position) throws IOException {
*
* @throws FileTransferException
*/
private void writeBlock(final byte[] buffer, final int offset,
final int length) throws FileTransferException {
private synchronized void writeBlock(final byte[] buffer, final int offset,
final int length)
throws FileTransferException {
if (length > 0 && !isReady) {
throw new FileTransferException(NO_FILE_IS_READY);
}
Expand Down Expand Up @@ -597,8 +600,9 @@ private void writeBlock(final byte[] buffer, final int offset,
*
* @throws FileTransferException
*/
private void writeBlockEnd(final byte[] buffer, final int offset,
final int length) throws FileTransferException {
private synchronized void writeBlockEnd(final byte[] buffer, final int offset,
final int length)
throws FileTransferException {
writeBlock(buffer, offset, length);
try {
closeFile();
Expand Down Expand Up @@ -629,7 +633,7 @@ private void checkByteBufSize(final int size) {
* @throws FileTransferException
* @throws FileEndOfTransferException
*/
private DataBlock getByteBlock(final int sizeblock)
private synchronized DataBlock getByteBlock(final int sizeblock)
throws FileTransferException, FileEndOfTransferException {
if (!isReady) {
throw new FileTransferException(NO_FILE_IS_READY);
Expand All @@ -656,7 +660,7 @@ private DataBlock getByteBlock(final int sizeblock)
* @throws FileTransferException
* @throws FileEndOfTransferException
*/
private DataBlock getByteBlock(final byte[] bufferGiven)
private synchronized DataBlock getByteBlock(final byte[] bufferGiven)
throws FileTransferException, FileEndOfTransferException {
if (!isReady) {
throw new FileTransferException(NO_FILE_IS_READY);
Expand Down
2 changes: 1 addition & 1 deletion WaarpCommon/src/main/java/org/waarp/common/guid/GUID.java
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ public GUID(final int tenantId, final int platformId) {
count = ++counter;
if (count > 0xFFFFFF) {
try {
FORSYNC.wait(1);//NOSONAR
Thread.sleep(1);//NOSONAR
} catch (final InterruptedException e) {//NOSONAR
// ignore
SysErrLogger.FAKE_LOGGER.ignoreLog(e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,14 @@
import io.netty.channel.WriteBufferWaterMark;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.util.IllegalReferenceCountException;
import io.netty.util.concurrent.Future;
import org.waarp.common.logging.SysErrLogger;

import java.io.IOException;
import java.net.InetAddress;
import java.net.ServerSocket;

/**
* Utility class for Netty usage
*/
Expand Down Expand Up @@ -159,6 +164,58 @@ public static void setServerBootstrap(final ServerBootstrap bootstrap,
}
}

/**
* Checks to see if a specific port is available.
*
* @param port the port to check for availability
*/
public static boolean availablePort(final int port) {
ServerSocket ss = null;
try {
ss = new ServerSocket(port); // NOSONAR
ss.setReuseAddress(true);
return true;
} catch (final IOException e) {
SysErrLogger.FAKE_LOGGER.ignoreLog(e);
} finally {
if (ss != null) {
try {
ss.close();
} catch (final Exception e) {
SysErrLogger.FAKE_LOGGER.ignoreLog(e);
}
}
}
return false;
}

/**
* Checks to see if a specific port is available.
*
* @param port the port to check for availability
* @param localAddress the associated localAddress to use
*/
public static boolean availablePort(final int port,
final InetAddress localAddress) {
ServerSocket ss = null;
try {
ss = new ServerSocket(port, 0, localAddress); // NOSONAR
ss.setReuseAddress(true);
return true;
} catch (final IOException e) {
SysErrLogger.FAKE_LOGGER.ignoreLog(e);
} finally {
if (ss != null) {
try {
ss.close();
} catch (final Exception e) {
SysErrLogger.FAKE_LOGGER.ignoreLog(e);
}
}
}
return false;
}

/**
* @param future
*
Expand Down Expand Up @@ -225,7 +282,12 @@ public static boolean release(final ByteBuf byteBuf) {
if (byteBuf == null || byteBuf.refCnt() <= 0) {
return true;
}
return byteBuf.release();
try {
return byteBuf.release();
} catch (final IllegalReferenceCountException e) {
SysErrLogger.FAKE_LOGGER.ignoreLog(e);
return true;
}
}

/**
Expand All @@ -236,7 +298,26 @@ public static boolean release(final ByteBuf byteBuf) {
public static void releaseCompletely(final ByteBuf byteBuf) {
if (byteBuf != null && byteBuf.refCnt() != 0) {
final int refCnt = byteBuf.refCnt();
byteBuf.release(refCnt);
try {
byteBuf.release(refCnt);
} catch (final IllegalReferenceCountException e) {
SysErrLogger.FAKE_LOGGER.ignoreLog(e);
}
}
}

/**
* Retain this ByteBuf
*
* @param byteBuf
*/
public static void retain(final ByteBuf byteBuf) {
if (byteBuf != null) {
try {
byteBuf.retain();
} catch (final IllegalReferenceCountException e) {
SysErrLogger.FAKE_LOGGER.ignoreLog(e);
}
}
}

Expand Down
Loading

0 comments on commit cbf8466

Please sign in to comment.