Skip to content

Commit 4972f68

Browse files
raydacraydac
committed
improved BINFILE to support deflate compression
1 parent 27f5364 commit 4972f68

File tree

5 files changed

+141
-117
lines changed

5 files changed

+141
-117
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
- MAVEN: added `ignoreMissingSources` boolean parameter, allows to skip preprocessing if source folders not found or not provided [#12](https://github.com/raydac/java-comment-preprocessor/issues/12)
1515
- MAVEN: added `skip` boolean parameter, it allows to skip execution, also it is possible to use `-Djcp.preprocess.skip=true` [#13](https://github.com/raydac/java-comment-preprocessor/issues/13)
1616
- CORE: added function `BOOL is(STR,ANY)` to check existence of variable for its name and compare its value with etalon (through string conversion, it will ensure true for `true` and `"true"` case) [#10](https://github.com/raydac/java-comment-preprocessor/issues/10)
17-
- CORE: improved the BINFILE function to support byte arrays in Golang, added types `uint8[]`, `uint8[]s`, `int8[]` and `int8[]s` (`s` means splitting to lines to avoid too long text line)
17+
- CORE: improved the BINFILE function, it allows `base64|byte[]|uint8[]|int8` and modifiers `s|d|ds|sd` where s - means splitting to lines and d - means deflate compression
1818

1919
- **6.1.0**
2020
- added `--es` option to enable spaces between comment chars and directives [#9](https://github.com/raydac/java-comment-preprocessor/issues/9), in ANT and MAVEN plugins it is boolean parameter named `allowWhitespace`, __NB! by default it is turned off for back compatibility!__

‎changelog.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
- MAVEN: added 'ignoreMissingSources' boolean parameter, allows to skip preprocessing if source folders not found or not provided #12
55
- MAVEN: added 'skip'boolean parameter, it allows to skip execution, also it is possible to use `-Djcp.preprocess.skip=true` #13
66
- CORE: added function `BOOL is(STR,ANY)` to check existence of variable for its name and compare its value with etalon (through string conversion, it will ensure true for `true` and `"true"` case), #10
7-
- CORE: improved the BINFILE function to support byte arrays in Golang, added types `uint8[]`, `uint8[]s`, `int8[]` and `int8[]s` (`s` means splitting to lines to avoid too long text line)
7+
- CORE: improved the BINFILE function, it allows `base64|byte[]|uint8[]|int8` and modifiers `s|d|ds|sd` where s - means splitting to lines and d - means deflate compression
88

99
6.1.0 (03 jul 2016)
1010
- implemented request #9, added support of whitespace between directive and comment, in command line it is `--es` option and in MAVEN and ANT it is boolean parameter `allowWhitespace`, by default it is turned off

‎src/main/java/com/igormaznitsa/jcp/expression/functions/FunctionBINFILE.java‎

Lines changed: 89 additions & 113 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.io.*;
2323

2424
import java.util.Locale;
25+
import java.util.zip.Deflater;
2526
import javax.annotation.Nonnull;
2627

2728
import javax.annotation.Nullable;
@@ -42,13 +43,9 @@ public class FunctionBINFILE extends AbstractFunction {
4243

4344
private enum Type {
4445
BASE64("base64"),
45-
BASE64_SPLITTED("base64s"),
4646
BYTEARRAY("byte[]"),
47-
BYTEARRAY_SPLITTED("byte[]s"),
4847
UINT8("uint8[]"),
49-
UINT8_SPLITTED("uint8[]s"),
50-
INT8("int8[]"),
51-
INT8_SPLITTED("int8[]s");
48+
INT8("int8[]");
5249

5350
private final String name;
5451

@@ -67,7 +64,7 @@ public static Type find(@Nullable final String name) {
6764
if (name != null) {
6865
final String normalized = name.toLowerCase(Locale.ENGLISH).trim();
6966
for (final Type t : values()) {
70-
if (t.name.equals(name)) {
67+
if (name.startsWith(t.name)) {
7168
result = t;
7269
break;
7370
}
@@ -77,6 +74,16 @@ public static Type find(@Nullable final String name) {
7774
}
7875
}
7976

77+
private static boolean hasSplitFlag(@Nonnull final String name, @Nonnull final Type type) {
78+
final String opts = name.substring(type.name.length());
79+
return opts.contains("S") || opts.contains("s");
80+
}
81+
82+
private static boolean hasDeflateFlag(@Nonnull final String name, @Nonnull final Type type) {
83+
final String opts = name.substring(type.name.length());
84+
return opts.contains("D") || opts.contains("d");
85+
}
86+
8087
@Override
8188
@Nonnull
8289
public String getName() {
@@ -87,11 +94,14 @@ public String getName() {
8794
@Nonnull
8895
public String getReference() {
8996
final StringBuilder buffer = new StringBuilder();
90-
for(final Type t : Type.values()){
91-
if (buffer.length()>0)buffer.append('|');
97+
for (final Type t : Type.values()) {
98+
if (buffer.length() > 0) {
99+
buffer.append('|');
100+
}
92101
buffer.append(t.name);
93102
}
94-
return "encode bin file into string representation, allowed types ["+buffer.toString()+']';
103+
buffer.append("[s|d|sd|ds]");
104+
return "encode bin file into string representation, allowed types [" + buffer.toString() + "], s - split to lines, d - deflater compression";
95105
}
96106

97107
@Override
@@ -115,16 +125,21 @@ public ValueType getResultType() {
115125
@Nonnull
116126
public Value executeStrStr(@Nonnull final PreprocessorContext context, @Nonnull final Value strfilePath, @Nonnull final Value encodeType) {
117127
final String filePath = strfilePath.asString();
118-
final Type type = Type.find(encodeType.asString());
128+
final String encodeTypeAsString = encodeType.asString();
129+
final Type type = Type.find(encodeTypeAsString);
119130

120131
if (type == null) {
121132
throw context.makeException("Unsupported encode type [" + encodeType.asString() + ']', null);
122133
}
123134

135+
final int lengthOfLine = hasSplitFlag(encodeTypeAsString, type) ? 80 : -1;
136+
final boolean doDeflate = hasDeflateFlag(encodeTypeAsString, type);
137+
124138
final File theFile;
125139
try {
126140
theFile = context.findFileInSourceFolder(filePath);
127-
} catch (IOException ex) {
141+
}
142+
catch (IOException ex) {
128143
throw context.makeException("Can't find bin file '" + filePath + '\'', null);
129144
}
130145

@@ -134,133 +149,94 @@ public Value executeStrStr(@Nonnull final PreprocessorContext context, @Nonnull
134149

135150
try {
136151
final String endOfLine = System.getProperty("line.separator", "\r\n");
137-
final String result;
138-
139-
switch (type) {
140-
case BASE64: {
141-
result = convertToBase64(theFile, -1, endOfLine).trim();
142-
}
143-
break;
144-
case BASE64_SPLITTED: {
145-
result = convertToBase64(theFile, 80, endOfLine).trim();
146-
}
147-
break;
148-
case BYTEARRAY: {
149-
result = convertToJBytes(theFile, -1, endOfLine);
150-
}
151-
break;
152-
case BYTEARRAY_SPLITTED: {
153-
result = convertToJBytes(theFile, 80, endOfLine);
154-
}
155-
break;
156-
case UINT8: {
157-
result = convertToUINT8(theFile, -1, endOfLine);
158-
}
159-
break;
160-
case UINT8_SPLITTED: {
161-
result = convertToUINT8(theFile, 80, endOfLine);
162-
}
163-
break;
164-
case INT8: {
165-
result = convertToINT8(theFile, -1, endOfLine);
166-
}
167-
break;
168-
case INT8_SPLITTED: {
169-
result = convertToINT8(theFile, 80, endOfLine);
170-
}
171-
break;
172-
default:
173-
throw new Error("Unexpected type : " + type);
174-
}
175-
return Value.valueOf(result);
176-
} catch (Exception ex) {
152+
return Value.valueOf(convertTo(theFile, type, doDeflate, lengthOfLine, endOfLine));
153+
}
154+
catch (Exception ex) {
177155
throw context.makeException("Unexpected exception", ex);
178156
}
179157
}
180158

181159
@Nonnull
182-
private String convertToUINT8(@Nonnull final File file, final int lineLength, @Nonnull final String endOfLine) throws IOException {
160+
private static String convertTo(@Nonnull final File file, @Nonnull final Type type, final boolean deflate, final int lineLength, @Nonnull final String endOfLine) throws IOException {
183161
final StringBuilder result = new StringBuilder(512);
184-
final byte[] array = FileUtils.readFileToByteArray(file);
162+
byte[] array = FileUtils.readFileToByteArray(file);
185163

186-
int endLinePos = lineLength;
164+
if (deflate) {
165+
array = deflate(array);
166+
}
187167

168+
int endLinePos = lineLength;
188169
boolean addNextLine = false;
189170

190-
for (final byte b : array) {
191-
if (addNextLine) {
192-
addNextLine = false;
193-
result.append(endOfLine);
171+
switch (type) {
172+
case BASE64: {
173+
final String baseEncoded = new Base64(lineLength, endOfLine.getBytes("UTF-8"), false).encodeAsString(array);
174+
result.append(baseEncoded.trim());
194175
}
195-
if (result.length() > 0) {
196-
result.append(',');
197-
}
198-
result.append(Integer.toString(b & 0xFF).toUpperCase(Locale.ENGLISH));
199-
if (lineLength > 0 && result.length() >= endLinePos) {
200-
addNextLine = true;
201-
endLinePos = result.length() + lineLength;
202-
}
203-
}
204-
205-
return result.toString();
206-
}
207-
208-
@Nonnull
209-
private String convertToINT8(@Nonnull final File file, final int lineLength, @Nonnull final String endOfLine) throws IOException {
210-
final StringBuilder result = new StringBuilder(512);
211-
final byte[] array = FileUtils.readFileToByteArray(file);
176+
break;
177+
case BYTEARRAY:
178+
case INT8:
179+
case UINT8: {
180+
for (final byte b : array) {
181+
if (addNextLine) {
182+
addNextLine = false;
183+
result.append(endOfLine);
184+
}
185+
186+
if (result.length() > 0) {
187+
result.append(',');
188+
}
212189

213-
int endLinePos = lineLength;
190+
switch (type) {
191+
case BYTEARRAY: {
192+
result.append("(byte)0x").append(Integer.toHexString(b & 0xFF).toUpperCase(Locale.ENGLISH));
193+
}
194+
break;
195+
case UINT8: {
196+
result.append(Integer.toString(b & 0xFF).toUpperCase(Locale.ENGLISH));
197+
}
198+
break;
199+
case INT8: {
200+
result.append(Integer.toString(b).toUpperCase(Locale.ENGLISH));
201+
}
202+
break;
203+
default:
204+
throw new Error("Unexpected type : " + type);
205+
}
214206

215-
boolean addNextLine = false;
207+
if (lineLength > 0 && result.length() >= endLinePos) {
208+
addNextLine = true;
209+
endLinePos = result.length() + lineLength;
210+
}
211+
}
216212

217-
for (final byte b : array) {
218-
if (addNextLine) {
219-
addNextLine = false;
220-
result.append(endOfLine);
221-
}
222-
if (result.length() > 0) {
223-
result.append(',');
224-
}
225-
result.append(Integer.toString(b));
226-
if (lineLength > 0 && result.length() >= endLinePos) {
227-
addNextLine = true;
228-
endLinePos = result.length() + lineLength;
229213
}
214+
break;
215+
default:
216+
throw new Error("Unexpected type : " + type);
230217
}
231218

232219
return result.toString();
233220
}
234221

235222
@Nonnull
236-
private String convertToJBytes(@Nonnull final File file, final int lineLength, @Nonnull final String endOfLine) throws IOException {
237-
final StringBuilder result = new StringBuilder(512);
238-
final byte[] array = FileUtils.readFileToByteArray(file);
223+
private static byte[] deflate(@Nonnull final byte[] data) throws IOException {
224+
final Deflater deflater = new Deflater(Deflater.BEST_COMPRESSION);
225+
deflater.setInput(data);
239226

240-
int endLinePos = lineLength;
241-
242-
boolean addNextLine = false;
227+
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(data.length);
243228

244-
for (final byte b : array) {
245-
if (addNextLine) {
246-
addNextLine = false;
247-
result.append(endOfLine);
248-
}
249-
if (result.length() > 0) {
250-
result.append(',');
251-
}
252-
result.append("(byte)0x").append(Integer.toHexString(b & 0xFF).toUpperCase(Locale.ENGLISH));
253-
if (lineLength > 0 && result.length() >= endLinePos) {
254-
addNextLine = true;
255-
endLinePos = result.length() + lineLength;
256-
}
229+
deflater.finish();
230+
final byte[] buffer = new byte[1024];
231+
while (!deflater.finished()) {
232+
final int count = deflater.deflate(buffer);
233+
outputStream.write(buffer, 0, count);
257234
}
235+
outputStream.close();
236+
final byte[] output = outputStream.toByteArray();
258237

259-
return result.toString();
260-
}
238+
deflater.end();
261239

262-
@Nonnull
263-
private String convertToBase64(@Nonnull final File file, final int lineLength, @Nonnull final String lineSeparator) throws IOException {
264-
return new Base64(lineLength, lineSeparator.getBytes("UTF-8"), false).encodeAsString(FileUtils.readFileToByteArray(file));
240+
return output;
265241
}
266242
}

‎src/test/java/com/igormaznitsa/jcp/expression/functions/FunctionBINFILETest.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,15 @@ public void testExecution_Base64Encoding() throws Exception {
3333
assertDestinationFolderEmpty();
3434
}
3535

36+
@Test
37+
public void testExecution_Base64Encoding_Deflate() throws Exception {
38+
final PreprocessorContext context = preparePreprocessorContext(getCurrentTestFolder());
39+
context.setLocalVariable("hello_world", Value.valueOf("Hello World!"));
40+
final Value result = Expression.evalExpression("binfile(\"./eval/TestBin.txt\",\"base64d\")", context);
41+
assertEquals("eNrzSM3JyVcIKEotKMpPTi0uzi9SBABHuwc9", result.asString().trim());
42+
assertDestinationFolderEmpty();
43+
}
44+
3645
@Test
3746
public void testExecution_Base64EncodingSplitted() throws Exception {
3847
final PreprocessorContext context = preparePreprocessorContext(getCurrentTestFolder());
@@ -79,6 +88,15 @@ public void testExecution_INT8ArrayEncoding() throws Exception {
7988
assertDestinationFolderEmpty();
8089
}
8190

91+
@Test
92+
public void testExecution_DEFLATEINT8ArrayEncoding() throws Exception {
93+
final PreprocessorContext context = preparePreprocessorContext(getCurrentTestFolder());
94+
context.setLocalVariable("hello_world", Value.valueOf("Hello World!"));
95+
final Value result = Expression.evalExpression("binfile(\"./eval/TestBinLong.txt\",\"int8[]d\")", context);
96+
assertEquals("120,-38,77,-116,49,10,-62,64,16,69,-5,-100,-30,-25,2,-98,67,59,-81,16,-78,19,93,-104,100,100,103,35,-40,25,45,-67,-116,10,-126,-92,-16,12,51,55,50,-111,20,54,-97,-57,-121,-9,108,-76,-69,-115,-10,-74,23,-4,98,31,-8,-39,-81,62,-8,-80,124,-10,-16,-21,-124,79,-65,21,107,98,22,108,19,29,-110,-44,-92,42,-87,68,-52,-120,-118,35,-91,-45,-33,-80,116,59,112,-20,8,89,-48,68,102,-28,61,-51,64,37,84,90,-54,-79,-91,-59,-20,-107,-102,-98,-47,72,66,38,-51,90,22,21,43,54,8,84,-57,64,97,14,84,33,64,-90,86,43,-119,126,-43,-43,23,-66,-35,76,-84", result.asString().trim());
97+
assertDestinationFolderEmpty();
98+
}
99+
82100
@Test
83101
public void testExecution_UINT8ArrayEncodingSplitted() throws Exception {
84102
final PreprocessorContext context = preparePreprocessorContext(getCurrentTestFolder());
@@ -115,6 +133,22 @@ public void testExecution_INT8ArrayEncodingSplitted() throws Exception {
115133
assertDestinationFolderEmpty();
116134
}
117135

136+
@Test
137+
public void testExecution_DEFLATEINT8ArrayEncodingSplitted() throws Exception {
138+
final PreprocessorContext context = preparePreprocessorContext(getCurrentTestFolder());
139+
context.setLocalVariable("hello_world", Value.valueOf("Hello World!"));
140+
final Value result = Expression.evalExpression("binfile(\"./eval/TestBinLong.txt\",\"int8[]ds\")", context);
141+
final String eof = System.getProperty("line.separator");
142+
assertEquals("120,-38,77,-116,49,10,-62,64,16,69,-5,-100,-30,-25,2,-98,67,59,-81,16,-78,19,93,-104" + eof
143+
+ ",100,100,103,35,-40,25,45,-67,-116,10,-126,-92,-16,12,51,55,50,-111,20,54,-97,-57" + eof
144+
+ ",-121,-9,108,-76,-69,-115,-10,-74,23,-4,98,31,-8,-39,-81,62,-8,-80,124,-10,-16,-21" + eof
145+
+ ",-124,79,-65,21,107,98,22,108,19,29,-110,-44,-92,42,-87,68,-52,-120,-118,35,-91" + eof
146+
+ ",-45,-33,-80,116,59,112,-20,8,89,-48,68,102,-28,61,-51,64,37,84,90,-54,-79,-91,-59" + eof
147+
+ ",-20,-107,-102,-98,-47,72,66,38,-51,90,22,21,43,54,8,84,-57,64,97,14,84,33,64,-90" + eof
148+
+ ",86,43,-119,126,-43,-43,23,-66,-35,76,-84", result.asString().trim());
149+
assertDestinationFolderEmpty();
150+
}
151+
118152
@Test
119153
public void testExecution_Str_wrongCases() throws Exception {
120154
assertFunctionException("binfile()");

‎src/test/java/com/igormaznitsa/jcp/usecases/AbstractUseCaseTest.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,8 +89,18 @@ private void assertFolder(final File folder1, final File folder2) throws Excepti
8989
}else{
9090
final boolean equalsLength = f.length() == f2.length();
9191
if (!equalsLength){
92-
System.err.println("WRONG FILE CONTENT\r\n---------------\r\n"+FileUtils.readFileToString(f2, "UTF-8")+"\r\n---------------");
93-
fail("Wrong length of result file (" + f.getName() + ", expected length = "+f2.length()+" but detected "+f.length());
92+
final String fileOne = FileUtils.readFileToString(f, "UTF-8");
93+
final String fileTwo = FileUtils.readFileToString(f2, "UTF-8");
94+
95+
System.err.println("FILE ONE=====================");
96+
System.err.println(fileOne);
97+
System.err.println("=============================");
98+
99+
System.err.println("FILE TWO=====================");
100+
System.err.println(fileTwo);
101+
System.err.println("=============================");
102+
103+
assertEquals("File content must be same", fileOne,fileTwo);
94104
}
95105
assertEquals("Checksum must be equal ("+f.getName()+')',FileUtils.checksumCRC32(f),FileUtils.checksumCRC32(f2));
96106
}
@@ -101,6 +111,10 @@ protected void tuneContext(final PreprocessorContext context){
101111

102112
}
103113

114+
private static void printDifference(final String etalon, final String value) {
115+
116+
}
117+
104118
@Test
105119
public final void main() throws Exception {
106120
final PreprocessorContext context = new PreprocessorContext();

0 commit comments

Comments
 (0)