Date: Thu, 13 Mar 2025 11:52:43 +0800
Subject: Add KAE zip feature
make/autoconf/libraries.m4 | 4 ++
make/autoconf/spec.gmk.in | 1 +
make/modules/java.base/lib/CoreLibraries.gmk | 39 ++++++++++--
src/hotspot/share/runtime/arguments.cpp | 9 +++
.../share/classes/java/util/zip/Deflater.java | 27 ++++++++
.../java/util/zip/GZIPInputStream.java | 62 ++++++++++++++++++-
.../java/util/zip/GZIPOutputStream.java | 35 ++++++++++-
.../share/classes/java/util/zip/Inflater.java | 23 +++++++
.../java/util/zip/InflaterInputStream.java | 21 +++++++
src/java.base/share/conf/security/java.policy | 4 ++
src/java.base/share/native/libzip/Deflater.c | 38 ++++++++++++
src/java.base/share/native/libzip/Inflater.c | 37 ++++++++++-
.../zip/GZIP/GZIPOutputStreamHeaderTest.java | 4 ++
13 files changed, 296 insertions(+), 8 deletions(-)
@@ -210,4 +210,8 @@ AC_DEFUN_ONCE([LIB_SETUP_MISC_LIBS],
# Control if libzip can use mmap. Available for purposes of overriding.
LIBZIP_CAN_USE_MMAP=true
AC_SUBST(LIBZIP_CAN_USE_MMAP)
+
+ # Control if libz can use mmap. Available for purposes of overriding.
+ LIBZ_CAN_USE_MMAP=true
+ AC_SUBST(LIBZ_CAN_USE_MMAP)
])
@@ -811,6 +811,7 @@ USE_EXTERNAL_LIBZ:=@USE_EXTERNAL_LIBZ@
LIBZ_CFLAGS:=@LIBZ_CFLAGS@
LIBZ_LIBS:=@LIBZ_LIBS@
LIBZIP_CAN_USE_MMAP:=@LIBZIP_CAN_USE_MMAP@
+LIBZ_CAN_USE_MMAP:=@LIBZ_CAN_USE_MMAP@
MSVCR_DLL:=@MSVCR_DLL@
VCRUNTIME_1_DLL:=@VCRUNTIME_1_DLL@
MSVCP_DLL:=@MSVCP_DLL@
@@ -84,9 +84,39 @@ $(BUILD_LIBJAVA): $(BUILD_LIBVERIFY)
##########################################################################################
-BUILD_LIBZIP_EXCLUDES :=
-ifeq ($(USE_EXTERNAL_LIBZ), true)
- LIBZIP_EXCLUDES += zlib
+ifeq ($(USE_EXTERNAL_LIBZ), false)
+ ifeq ($(OPENJDK_TARGET_OS), linux)
+ ifeq ($(LIBZ_CAN_USE_MMAP), true)
+ BUILD_LIBZ_MMAP := -DUSE_MMAP
+ endif
+
+ $(eval $(call SetupJdkLibrary, BUILD_LIBZ, \
+ NAME := z, \
+ SRC := $(TOPDIR)/src/java.base/share/native/libzip/zlib, \
+ OPTIMIZATION := LOW, \
+ CFLAGS := $(filter-out -fvisibility=hidden,$(CFLAGS_JDKLIB) $(LIBZ_CFLAGS)), \
+ CFLAGS_unix := $(BUILD_LIBZ_MMAP) -UDEBUG, \
+ DISABLED_WARNINGS_gcc := unused-function implicit-fallthrough, \
+ DISABLED_WARNINGS_gcc_gzlib.c := implicit-function-declaration, \
+ DISABLED_WARNINGS_gcc_gzwrite.c := implicit-function-declaration, \
+ DISABLED_WARNINGS_gcc_gzread.c := implicit-function-declaration, \
+ DISABLED_WARNINGS_clang := format-nonliteral, \
+ LDFLAGS := $(LDFLAGS_JDKLIB) \
+ $(call SET_SHARED_LIBRARY_ORIGIN), \
+ LIBS_unix := , \
+ LIBS_windows := jvm.lib $(WIN_JAVA_LIB), \
+ ))
+
+ $(BUILD_LIBZ): $(BUILD_LIBJAVA)
+
+ TARGETS += $(BUILD_LIBZ)
+ endif
+endif
+
+##########################################################################################
+
+ifeq ($(OPENJDK_TARGET_OS), linux)
+ LIBZIP_EXCLUDES := zlib
endif
ifeq ($(LIBZIP_CAN_USE_MMAP), true)
@@ -105,10 +135,11 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBZIP, \
DISABLED_WARNINGS_clang_gzwrite.c := format-nonliteral, \
LDFLAGS := $(LDFLAGS_JDKLIB) \
$(call SET_SHARED_LIBRARY_ORIGIN), \
- LIBS_unix := -ljvm -ljava $(LIBZ_LIBS), \
+ LIBS_unix := -ljvm -ljava -lz, \
LIBS_windows := jvm.lib $(WIN_JAVA_LIB), \
))
+$(BUILD_LIBZIP): $(BUILD_LIBZ)
$(BUILD_LIBZIP): $(BUILD_LIBJAVA)
TARGETS += $(BUILD_LIBZIP)
@@ -2551,6 +2551,15 @@ jint Arguments::parse_each_vm_init_arg(const JavaVMInitArgs* args, bool* patch_m
// -D
} else if (match_option(option, "-D", &tail)) {
const char* value;
+#ifndef AARCH64
+ if (match_option(option, "-DGZIP_USE_KAE=", &value)) {
+ if (strcmp(value, "true") == 0) {
+ jio_fprintf(defaultStream::output_stream(),
+ "-DGZIP_USE_KAE is not supported. This system propertiy is valid only on aarch64 architecture machines.\n"
+ "The compression action is performed using the native compression capability of the JDK.\n");
+ }
+ }
+#endif
if (match_option(option, "-Djava.endorsed.dirs=", &value) &&
*value!= '\0' && strcmp(value, "\"\"") != 0) {
// abort if -Djava.endorsed.dirs is set
@@ -203,6 +203,20 @@ public class Deflater {
init(level, DEFAULT_STRATEGY, nowrap));
}
+ /**
+ * Creates a new compressor using the specified compression level
+ * and windowBits.
+ * This method is mainly used to support the KAE-zip feature.
+ * @param level the compression level (0-9)
+ * @param windowBits compression format (-15~31)
+ */
+ public Deflater(int level, int windowBits) {
+ this.level = level;
+ this.strategy = DEFAULT_STRATEGY;
+ this.zsRef = new DeflaterZStreamRef(this,
+ initKAE(level, DEFAULT_STRATEGY, windowBits));
+ }
+
/**
* Creates a new compressor using the specified compression level.
* Compressed data will be generated in ZLIB format.
@@ -882,6 +896,18 @@ public class Deflater {
}
}
+ /**
+ * Resets deflater so that a new set of input data can be processed.
+ * Java fields are not initialized.
+ * This method is mainly used to support the KAE-zip feature.
+ */
+ public void resetKAE() {
+ synchronized (zsRef) {
+ ensureOpen();
+ reset(zsRef.address());
+ }
+ }
+
/**
* Closes the compressor and discards any unprocessed input.
*
@@ -913,6 +939,7 @@ public class Deflater {
}
private static native long init(int level, int strategy, boolean nowrap);
+ private static native long initKAE(int level, int strategy, int windowBits);
private static native void setDictionary(long addr, byte[] b, int off,
int len);
private static native void setDictionaryBuffer(long addr, long bufAddress, int len);
@@ -54,6 +54,23 @@ public class GZIPInputStream extends InflaterInputStream {
private boolean closed = false;
+ /**
+ * The field is mainly used to support the KAE-zip feature.
+ */
+ private static boolean GZIP_USE_KAE = false;
+
+ private static int WINDOWBITS = 31;
+
+ private static int FLUSHKAE = 2;
+
+ static {
+ if ("aarch64".equals(System.getProperty("os.arch"))) {
+ GZIP_USE_KAE = Boolean.parseBoolean(System.getProperty("GZIP_USE_KAE", "false"));
+ WINDOWBITS = Integer.parseInt(System.getProperty("WINDOWBITS", "31"));
+ FLUSHKAE = Integer.parseInt(System.getProperty("FLUSHKAE", "2"));
+ }
+ }
+
/**
* Check to make sure that this stream has not been closed
*/
@@ -77,6 +77,11 @@ public class GZIPInputStream extends InflaterInputStream {
public GZIPInputStream(InputStream in, int size) throws IOException {
super(in, createInflater(in, size), size);
usesDefaultInflater = true;
+
+ // When GZIP_USE_KAE is true, the header of the file is readed
+ // through the native zlib library, not in java code.
+ if (GZIP_USE_KAE) return;
+
try {
readHeader(in);
} catch (IOException ioe) {
@@ -97,7 +102,7 @@ public class GZIPInputStream extends InflaterInputStream {
if (size <= 0) {
throw new IllegalArgumentException("buffer size <= 0");
}
- return new Inflater(true);
+ return GZIP_USE_KAE ? new Inflater(WINDOWBITS, FLUSHKAE) : new Inflater(true);
}
/**
@@ -127,13 +149,16 @@ public class GZIPInputStream extends InflaterInputStream {
}
int n = super.read(buf, off, len);
if (n == -1) {
- if (readTrailer())
+ if (GZIP_USE_KAE ? readTrailerKAE() : readTrailer())
eos = true;
else
return this.read(buf, off, len);
} else {
crc.update(buf, off, n);
}
+ if (GZIP_USE_KAE && inf.finished()) {
+ if (readTrailerKAE()) eos = true;
+ }
return n;
}
@@ -254,6 +279,39 @@ public class GZIPInputStream extends InflaterInputStream {
return false;
}
+ /*
+ * Reads GZIP member trailer and returns true if the eos
+ * reached, false if there are more (concatenated gzip
+ * data set)
+ *
+ * This method is mainly used to support the KAE-zip feature.
+ */
+ private boolean readTrailerKAE() throws IOException {
+ InputStream in = this.in;
+ int n = inf.getRemaining();
+ if (n > 0) {
+ in = new SequenceInputStream(
+ new ByteArrayInputStream(buf, len - n, n),
+ new FilterInputStream(in) {
+ public void close() throws IOException {}
+ });
+ }
+ // If there are more bytes available in "in" or the leftover in the "inf" is > 18 bytes:
+ // next.header.min(10) + next.trailer(8), try concatenated case
+
+ if (n > 18) {
+ inf.reset();
+ inf.setInput(buf, len - n, n);
+ } else {
+ try {
+ fillKAE(n);
+ } catch (IOException e) {
+ return true;
+ }
+ }
+ return false;
+ }
+
/*
* Reads unsigned integer in Intel byte order.
*/
@@ -55,6 +55,20 @@ public class GZIPOutputStream extends DeflaterOutputStream {
// Represents the default "unknown" value for OS header, per RFC-1952
private static final byte OS_UNKNOWN = (byte) 255;
+ /**
+ * The field is mainly used to support the KAE-zip feature.
+ */
+ private static boolean GZIP_USE_KAE = false;
+
+ private static int WINDOWBITS = 31;
+
+ static {
+ if ("aarch64".equals(System.getProperty("os.arch"))) {
+ GZIP_USE_KAE = Boolean.parseBoolean(System.getProperty("GZIP_USE_KAE", "false"));
+ WINDOWBITS = Integer.parseInt(System.getProperty("WINDOWBITS", "31"));
+ }
+ }
+
/**
* Creates a new output stream with the specified buffer size.
*
@@ -90,10 +104,16 @@ public class GZIPOutputStream extends DeflaterOutputStream {
public GZIPOutputStream(OutputStream out, int size, boolean syncFlush)
throws IOException
{
- super(out, out != null ? new Deflater(Deflater.DEFAULT_COMPRESSION, true) : null,
+ super(out, out != null ?
+ (GZIP_USE_KAE ? new Deflater(Deflater.DEFAULT_COMPRESSION, WINDOWBITS) :
+ new Deflater(Deflater.DEFAULT_COMPRESSION, true)) : null,
size,
syncFlush);
usesDefaultDeflater = true;
+
+ // When GZIP_USE_KAE is true, the header of the file is written
+ // through the native zlib library, not in java code.
+ if (GZIP_USE_KAE) return;
writeHeader();
crc.reset();
}
@@ -163,6 +183,13 @@ public class GZIPOutputStream extends DeflaterOutputStream {
int len = def.deflate(buf, 0, buf.length);
if (def.finished() && len <= buf.length - TRAILER_SIZE) {
// last deflater buffer. Fit trailer at the end
+ // When GZIP_USE_KAE is true, the trailer of the file is written
+ // through the native zlib library, not in java code.
+ if (GZIP_USE_KAE) {
+ out.write(buf, 0, len);
+ def.resetKAE();
+ return;
+ }
writeTrailer(buf, len);
len = len + TRAILER_SIZE;
out.write(buf, 0, len);
@@ -173,6 +200,12 @@ public class GZIPOutputStream extends DeflaterOutputStream {
}
// if we can't fit the trailer at the end of the last
// deflater buffer, we write it separately
+ // When GZIP_USE_KAE is true, the trailer of the file is written
+ // through the native zlib library, not in java code.
+ if (GZIP_USE_KAE) {
+ def.resetKAE();
+ return;
+ }
byte[] trailer = new byte[TRAILER_SIZE];
writeTrailer(trailer, 0);
out.write(trailer);
@@ -134,6 +134,17 @@ public class Inflater {
this.zsRef = new InflaterZStreamRef(this, init(nowrap));
}
+ /**
+ * Creates a new decompressor.
+ * This method is mainly used to support the KAE-zip feature.
+ *
+ * @param windowBits compression format (-15~31)
+ * @param flushKAE inflate flush type (0~6)
+ */
+ public Inflater(int windowBits, int flushKAE) {
+ this.zsRef = new InflaterZStreamRef(this, initKAE(windowBits, flushKAE));
+ }
+
/**
* Creates a new decompressor.
*/
@@ -702,6 +713,17 @@ public class Inflater {
}
}
+ /**
+ * Resets inflater so that a new set of input data can be processed.
+ * This method is mainly used to support the KAE-zip feature.
+ */
+ public void resetKAE() {
+ synchronized (zsRef) {
+ ensureOpen();
+ reset(zsRef.address());
+ }
+ }
+
/**
* Closes the decompressor and discards any unprocessed input.
*
@@ -730,6 +752,7 @@ public class Inflater {
private static native void initIDs();
private static native long init(boolean nowrap);
+ private static native long initKAE(int windowBits, int flushKAE);
private static native void setDictionary(long addr, byte[] b, int off,
int len);
private static native void setDictionaryBuffer(long addr, long bufAddress, int len);
@@ -268,6 +268,27 @@ public class InflaterInputStream extends FilterInputStream {
inf.setInput(buf, 0, len);
}
+ /**
+ * Fills input buffer with more data to decompress.
+ * This method is mainly used to support the KAE-zip feature.
+ * @param n Maximum Read Bytes
+ * @throws IOException if an I/O error has occurred
+ */
+ protected void fillKAE(int n) throws IOException {
+ ensureOpen();
+ byte[] buftmp = new byte[buf.length];
+ if (n != 0) {
+ System.arraycopy(buf, buf.length - n, buftmp, 0, n);
+ }
+ int kaelen = in.read(buftmp, n, buf.length - n);
+ if (kaelen == -1) {
+ throw new EOFException("Unexpected end of ZLIB input stream");
+ }
+ System.arraycopy(buftmp, 0, buf, buf.length - n - kaelen, n + kaelen);
+ inf.reset();
+ inf.setInput(buf, buf.length - n - kaelen, n + kaelen);
+ }
+
/**
* Tests if this input stream supports the {@code mark} and
* {@code reset} methods. The {@code markSupported}
@@ -43,4 +43,8 @@ grant {
permission java.util.PropertyPermission "java.vm.version", "read";
permission java.util.PropertyPermission "java.vm.vendor", "read";
permission java.util.PropertyPermission "java.vm.name", "read";
+
+ permission java.util.PropertyPermission "GZIP_USE_KAE", "read";
+ permission java.util.PropertyPermission "WINDOWBITS", "read";
+ permission java.util.PropertyPermission "FLUSHKAE", "read";
};
@@ -76,6 +76,44 @@ Java_java_util_zip_Deflater_init(JNIEnv *env, jclass cls, jint level,
}
}
+JNIEXPORT jlong JNICALL
+Java_java_util_zip_Deflater_initKAE(JNIEnv *env, jclass cls, jint level,
+ jint strategy, jint windowBits)
+{
+ z_stream *strm = calloc(1, sizeof(z_stream));
+
+ if (strm == 0) {
+ JNU_ThrowOutOfMemoryError(env, 0);
+ return jlong_zero;
+ } else {
+ const char *msg;
+ int ret = deflateInit2(strm, level, Z_DEFLATED,
+ windowBits,
+ DEF_MEM_LEVEL, strategy);
+ switch (ret) {
+ case Z_OK:
+ return ptr_to_jlong(strm);
+ case Z_MEM_ERROR:
+ free(strm);
+ JNU_ThrowOutOfMemoryError(env, 0);
+ return jlong_zero;
+ case Z_STREAM_ERROR:
+ free(strm);
+ JNU_ThrowIllegalArgumentException(env, 0);
+ return jlong_zero;
+ default:
+ msg = ((strm->msg != NULL) ? strm->msg :
+ (ret == Z_VERSION_ERROR) ?
+ "zlib returned Z_VERSION_ERROR: "
+ "compile time and runtime zlib implementations differ" :
+ "unknown error initializing zlib library");
+ free(strm);
+ JNU_ThrowInternalError(env, msg);
+ return jlong_zero;
+ }
+ }
+}
+
static void throwInternalErrorHelper(JNIEnv *env, z_stream *strm, const char *fixmsg) {
const char *msg = NULL;
msg = (strm->msg != NULL) ? strm->msg : fixmsg;
@@ -44,6 +44,7 @@
static jfieldID inputConsumedID;
static jfieldID outputConsumedID;
+static jint inflaterFlushType = Z_PARTIAL_FLUSH;
JNIEXPORT void JNICALL
Java_java_util_zip_Inflater_initIDs(JNIEnv *env, jclass cls)
@@ -87,6 +88,40 @@ Java_java_util_zip_Inflater_init(JNIEnv *env, jclass cls, jboolean nowrap)
}
}
+JNIEXPORT jlong JNICALL
+Java_java_util_zip_Inflater_initKAE(JNIEnv *env, jclass cls, jint windowBits, jint flushKAE)
+{
+ z_stream *strm = calloc(1, sizeof(z_stream));
+ inflaterFlushType = flushKAE;
+
+ if (strm == NULL) {
+ JNU_ThrowOutOfMemoryError(env, 0);
+ return jlong_zero;
+ } else {
+ const char *msg;
+ int ret = inflateInit2(strm, windowBits);
+ switch (ret) {
+ case Z_OK:
+ return ptr_to_jlong(strm);
+ case Z_MEM_ERROR:
+ free(strm);
+ JNU_ThrowOutOfMemoryError(env, 0);
+ return jlong_zero;
+ default:
+ msg = ((strm->msg != NULL) ? strm->msg :
+ (ret == Z_VERSION_ERROR) ?
+ "zlib returned Z_VERSION_ERROR: "
+ "compile time and runtime zlib implementations differ" :
+ (ret == Z_STREAM_ERROR) ?
+ "inflateInit2 returned Z_STREAM_ERROR" :
+ "unknown error initializing zlib library");
+ free(strm);
+ JNU_ThrowInternalError(env, msg);
+ return jlong_zero;
+ }
+ }
+}
+
static void checkSetDictionaryResult(JNIEnv *env, jlong addr, int res)
{
switch (res) {
@@ -137,7 +172,7 @@ static jint doInflate(jlong addr,
strm->avail_in = inputLen;
strm->avail_out = outputLen;
- ret = inflate(strm, Z_PARTIAL_FLUSH);
+ ret = inflate(strm, inflaterFlushType);
return ret;
}
@@ -35,6 +35,10 @@ import java.util.zip.GZIPOutputStream;
* @bug 8244706
* @summary Verify that the OS header flag in the stream written out by java.util.zip.GZIPOutputStream
* has the correct expected value
+ * @comment This test case is not suitable for GZIP-zip feature testing, the ninth byte in the header
+ * of the gzip file identifies the operating system that generated the file. By default, the byte of
+ * the compressed package generated by JDK is 0xff (OS_UNKNOWN). KAE-zip relies on the underlying
+ * zlib library to write header files, this field is set to 0x03 (OS_UNIX). Therefore, this case will fail
* @run testng GZIPOutputStreamHeaderTest
*/
public class GZIPOutputStreamHeaderTest {
--
2.47.0.windows.2
