From 78d1200608a89876f3e3a2e55bbf7580e643cff4 Mon Sep 17 00:00:00 2001 From: topjohnwu Date: Thu, 1 May 2025 02:28:00 -0700 Subject: [PATCH] Migrate all compression code to Rust --- .gitmodules | 3 - native/src/Android.mk | 4 +- native/src/Cargo.lock | 127 +++- native/src/Cargo.toml | 9 +- native/src/base/misc.rs | 58 ++ native/src/boot/Cargo.toml | 7 +- native/src/boot/bootimg.cpp | 78 +- native/src/boot/bootimg.hpp | 10 +- native/src/boot/bzlib.h | 287 ------- native/src/boot/compress.cpp | 762 ------------------- native/src/boot/compress.hpp | 14 - native/src/boot/compress.rs | 426 +++++++++++ native/src/boot/cpio.rs | 32 +- native/src/boot/format.cpp | 99 +-- native/src/boot/format.hpp | 41 +- native/src/boot/lib.rs | 39 +- native/src/boot/main.cpp | 101 ++- native/src/boot/payload.rs | 21 +- native/src/external/Android.mk | 21 - native/src/external/crt0 | 2 +- native/src/external/lz4-sys/Cargo.toml | 10 + native/src/external/lz4-sys/src/lib.rs | 480 ++++++++++++ native/src/external/lz4-sys/src/wasm_shim.rs | 123 +++ native/src/external/lzma-sys/Cargo.toml | 22 + native/src/external/lzma-sys/src/lib.rs | 359 +++++++++ native/src/external/zopfli | 1 - 26 files changed, 1871 insertions(+), 1265 deletions(-) delete mode 100644 native/src/boot/bzlib.h delete mode 100644 native/src/boot/compress.cpp delete mode 100644 native/src/boot/compress.hpp create mode 100644 native/src/boot/compress.rs create mode 100644 native/src/external/lz4-sys/Cargo.toml create mode 100644 native/src/external/lz4-sys/src/lib.rs create mode 100644 native/src/external/lz4-sys/src/wasm_shim.rs create mode 100644 native/src/external/lzma-sys/Cargo.toml create mode 100644 native/src/external/lzma-sys/src/lib.rs delete mode 160000 native/src/external/zopfli diff --git a/.gitmodules b/.gitmodules index 89f2f0886..acf506d3b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,9 +10,6 @@ [submodule "libcxx"] path = native/src/external/libcxx url = https://github.com/topjohnwu/libcxx.git -[submodule "zopfli"] - path = native/src/external/zopfli - url = https://github.com/google/zopfli.git [submodule "cxx-rs"] path = native/src/external/cxx-rs url = https://github.com/topjohnwu/cxx.git diff --git a/native/src/Android.mk b/native/src/Android.mk index c0df67078..27b2f77ef 100644 --- a/native/src/Android.mk +++ b/native/src/Android.mk @@ -85,13 +85,11 @@ LOCAL_STATIC_LIBRARIES := \ libbase \ liblzma \ liblz4 \ - libzopfli \ libboot-rs LOCAL_SRC_FILES := \ boot/main.cpp \ boot/bootimg.cpp \ - boot/compress.cpp \ boot/format.cpp \ boot/boot-rs.cpp @@ -99,7 +97,7 @@ LOCAL_LDFLAGS := -static ifdef B_CRT0 LOCAL_STATIC_LIBRARIES += crt0 -LOCAL_LDFLAGS += -lm +LOCAL_LDFLAGS += -lm -Wl,--defsym=vfprintf=musl_vfprintf endif include $(BUILD_EXECUTABLE) diff --git a/native/src/Cargo.lock b/native/src/Cargo.lock index a36cd1e29..9a9ebac9d 100644 --- a/native/src/Cargo.lock +++ b/native/src/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "anstyle" version = "1.0.10" @@ -95,6 +101,12 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "bumpalo" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" + [[package]] name = "bytemuck" version = "1.22.0" @@ -121,6 +133,26 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", + "libbz2-rs-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "cc" version = "1.2.19" @@ -207,6 +239,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-bigint" version = "0.6.1" @@ -390,6 +431,17 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" +[[package]] +name = "flate2" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" +dependencies = [ + "crc32fast", + "libz-rs-sys", + "miniz_oxide", +] + [[package]] name = "foldhash" version = "0.1.5" @@ -451,9 +503,6 @@ name = "libbz2-rs-sys" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0864a00c8d019e36216b69c2c4ce50b83b7bd966add3cf5ba554ec44f8bebcf5" -dependencies = [ - "libc", -] [[package]] name = "libc" @@ -476,6 +525,29 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +dependencies = [ + "libc", +] + +[[package]] +name = "lzma-sys" +version = "0.1.20" +dependencies = [ + "libc", +] + [[package]] name = "magisk" version = "0.0.0" @@ -502,13 +574,14 @@ dependencies = [ "block-buffer", "bytemuck", "byteorder", + "bzip2", "cxx", "cxx-gen", "der", "digest", "fdt", - "libbz2-rs-sys", - "libz-rs-sys", + "flate2", + "lz4", "num-traits", "p256", "p384", @@ -521,6 +594,8 @@ dependencies = [ "sha2", "size", "x509-cert", + "xz2", + "zopfli", ] [[package]] @@ -555,6 +630,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" +dependencies = [ + "adler2", +] + [[package]] name = "nom" version = "7.1.3" @@ -663,6 +747,12 @@ dependencies = [ "spki", ] +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "primefield" version = "0.14.0-pre.0" @@ -837,6 +927,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "size" version = "0.5.0" @@ -1050,6 +1146,15 @@ dependencies = [ "tls_codec", ] +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "zeroize" version = "1.8.1" @@ -1075,3 +1180,15 @@ name = "zlib-rs" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" + +[[package]] +name = "zopfli" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfc5ee405f504cd4984ecc6f14d02d55cfda60fa4b689434ef4102aae150cd7" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/native/src/Cargo.toml b/native/src/Cargo.toml index dee74e4d2..f1cc71792 100644 --- a/native/src/Cargo.toml +++ b/native/src/Cargo.toml @@ -25,10 +25,13 @@ syn = "2.0.100" quote = "1.0.40" proc-macro2 = "1.0.94" argh = { version = "0.1.13", default-features = false } -libz-rs-sys = { version = "0.5.0", features = ["export-symbols"] } -libbz2-rs-sys = { version = "0.1.3" } pb-rs = { version = "0.10.0", default-features = false } quick-protobuf = "0.8.1" +flate2 = { version = "1.1.1", default-features = false } +bzip2 = { version = "0.5.2", default-features = false } +zopfli = "0.8.2" +lz4 = "1.28.1" +xz2 = "0.1.7" # Rust crypto crates are tied together sha1 = "=0.11.0-pre.4" @@ -48,6 +51,8 @@ sec1 = "=0.8.0-rc.3" [patch.crates-io] pb-rs = { git = "https://github.com/tafia/quick-protobuf.git" } quick-protobuf = { git = "https://github.com/tafia/quick-protobuf.git" } +lz4-sys = { path = "external/lz4-sys" } +lzma-sys = { path = "external/lzma-sys" } [profile.dev] opt-level = "z" diff --git a/native/src/base/misc.rs b/native/src/base/misc.rs index 8685f9d8d..2fa2a1c9e 100644 --- a/native/src/base/misc.rs +++ b/native/src/base/misc.rs @@ -169,3 +169,61 @@ impl Default for AtomicArc { Self::new(Default::default()) } } + +pub struct Chunker { + chunk: Box<[u8]>, + chunk_size: usize, + pos: usize, +} + +impl Chunker { + pub fn new(chunk_size: usize) -> Self { + Chunker { + // SAFETY: all bytes will be initialized before it is used, tracked by self.pos + chunk: unsafe { Box::new_uninit_slice(chunk_size).assume_init() }, + chunk_size, + pos: 0, + } + } + + pub fn set_chunk_size(&mut self, chunk_size: usize) { + self.chunk_size = chunk_size; + self.pos = 0; + if self.chunk.len() < chunk_size { + self.chunk = unsafe { Box::new_uninit_slice(chunk_size).assume_init() }; + } + } + + // Returns (remaining buf, Option) + pub fn add_data<'a, 'b: 'a>(&'a mut self, mut buf: &'b [u8]) -> (&'b [u8], Option<&'a [u8]>) { + let mut chunk = None; + if self.pos > 0 { + // Try to fill the chunk + let len = std::cmp::min(self.chunk_size - self.pos, buf.len()); + self.chunk[self.pos..self.pos + len].copy_from_slice(&buf[..len]); + self.pos += len; + // If the chunk is filled, consume it + if self.pos == self.chunk_size { + chunk = Some(&self.chunk[..self.chunk_size]); + self.pos = 0; + } + buf = &buf[len..]; + } else if buf.len() >= self.chunk_size { + // Directly consume a chunk from buf + chunk = Some(&buf[..self.chunk_size]); + buf = &buf[self.chunk_size..]; + } else { + // Copy buf into chunk + self.chunk[self.pos..self.pos + buf.len()].copy_from_slice(buf); + self.pos += buf.len(); + return (&[], None); + } + (buf, chunk) + } + + pub fn get_available(&mut self) -> &[u8] { + let chunk = &self.chunk[..self.pos]; + self.pos = 0; + chunk + } +} diff --git a/native/src/boot/Cargo.toml b/native/src/boot/Cargo.toml index 065dd7754..29a31dac2 100644 --- a/native/src/boot/Cargo.toml +++ b/native/src/boot/Cargo.toml @@ -30,8 +30,11 @@ der = { workspace = true, features = ["derive", "pem"] } fdt = { workspace = true } bytemuck = { workspace = true, features = ["derive", "min_const_generics"] } num-traits = { workspace = true } -libz-rs-sys = { workspace = true } -libbz2-rs-sys = { workspace = true } +flate2 = { workspace = true, features = ["zlib-rs"] } +bzip2 = { workspace = true, features = ["libbz2-rs-sys"] } +lz4 = { workspace = true } +xz2 = { workspace = true } +zopfli = { workspace = true, features = ["gzip"] } # Pin version to prevent cargo update break builds block-buffer = { workspace = true } diff --git a/native/src/boot/bootimg.cpp b/native/src/boot/bootimg.cpp index 2a5cc9c16..d94823f3c 100644 --- a/native/src/boot/bootimg.cpp +++ b/native/src/boot/bootimg.cpp @@ -8,7 +8,6 @@ #include "boot-rs.hpp" #include "bootimg.hpp" #include "magiskboot.hpp" -#include "compress.hpp" using namespace std; @@ -16,17 +15,13 @@ using namespace std; #define SHA256_DIGEST_SIZE 32 #define SHA_DIGEST_SIZE 20 -static void decompress(format_t type, int fd, const void *in, size_t size) { - auto ptr = get_decoder(type, make_unique(fd)); - ptr->write(in, size); +static void decompress(FileFormat type, int fd, const void *in, size_t size) { + decompress_bytes(type, byte_view { in, size }, fd); } -static off_t compress(format_t type, int fd, const void *in, size_t size) { +static off_t compress_len(FileFormat type, byte_view in, int fd) { auto prev = lseek(fd, 0, SEEK_CUR); - { - auto strm = get_encoder(type, make_unique(fd)); - strm->write(in, size); - } + compress_bytes(type, in, fd); auto now = lseek(fd, 0, SEEK_CUR); return now - prev; } @@ -149,29 +144,30 @@ void dyn_img_hdr::load_hdr_file() { }); } -boot_img::boot_img(const char *image) : map(image) { +boot_img::boot_img(const char *image) : +map(image), k_fmt(FileFormat::UNKNOWN), r_fmt(FileFormat::UNKNOWN), e_fmt(FileFormat::UNKNOWN) { fprintf(stderr, "Parsing boot image: [%s]\n", image); for (const uint8_t *addr = map.buf(); addr < map.buf() + map.sz(); ++addr) { - format_t fmt = check_fmt(addr, map.sz()); + FileFormat fmt = check_fmt(addr, map.sz()); switch (fmt) { - case CHROMEOS: + case FileFormat::CHROMEOS: // chromeos require external signing flags[CHROMEOS_FLAG] = true; addr += 65535; break; - case DHTB: + case FileFormat::DHTB: flags[DHTB_FLAG] = true; flags[SEANDROID_FLAG] = true; fprintf(stderr, "DHTB_HDR\n"); addr += sizeof(dhtb_hdr) - 1; break; - case BLOB: + case FileFormat::BLOB: flags[BLOB_FLAG] = true; fprintf(stderr, "TEGRA_BLOB\n"); addr += sizeof(blob_hdr) - 1; break; - case AOSP: - case AOSP_VENDOR: + case FileFormat::AOSP: + case FileFormat::AOSP_VENDOR: if (parse_image(addr, fmt)) return; // fallthrough @@ -257,9 +253,9 @@ static int find_dtb_offset(const uint8_t *buf, unsigned sz) { return -1; } -static format_t check_fmt_lg(const uint8_t *buf, unsigned sz) { - format_t fmt = check_fmt(buf, sz); - if (fmt == LZ4_LEGACY) { +static FileFormat check_fmt_lg(const uint8_t *buf, unsigned sz) { + FileFormat fmt = check_fmt(buf, sz); + if (fmt == FileFormat::LZ4_LEGACY) { // We need to check if it is LZ4_LG uint32_t off = 4; uint32_t block_sz; @@ -267,7 +263,7 @@ static format_t check_fmt_lg(const uint8_t *buf, unsigned sz) { memcpy(&block_sz, buf + off, sizeof(block_sz)); off += sizeof(block_sz); if (off + block_sz > sz) - return LZ4_LG; + return FileFormat::LZ4_LG; off += block_sz; } } @@ -276,8 +272,8 @@ static format_t check_fmt_lg(const uint8_t *buf, unsigned sz) { #define CMD_MATCH(s) BUFFER_MATCH(h->cmdline, s) -pair boot_img::create_hdr(const uint8_t *addr, format_t type) { - if (type == AOSP_VENDOR) { +pair boot_img::create_hdr(const uint8_t *addr, FileFormat type) { + if (type == FileFormat::AOSP_VENDOR) { fprintf(stderr, "VENDOR_BOOT_HDR\n"); auto h = reinterpret_cast(addr); switch (h->header_version) { @@ -377,7 +373,7 @@ off += hdr->name##_size(); \ off = align_to(off, hdr->page_size()); \ assert_off(); -bool boot_img::parse_image(const uint8_t *p, format_t type) { +bool boot_img::parse_image(const uint8_t *p, FileFormat type) { auto [base_addr, hdr] = create_hdr(p, type); if (hdr == nullptr) { fprintf(stderr, "Invalid boot image header!\n"); @@ -419,7 +415,7 @@ bool boot_img::parse_image(const uint8_t *p, format_t type) { } k_fmt = check_fmt_lg(kernel, hdr->kernel_size()); - if (k_fmt == MTK) { + if (k_fmt == FileFormat::MTK) { fprintf(stderr, "MTK_KERNEL_HDR\n"); flags[MTK_KERNEL] = true; k_hdr = reinterpret_cast(kernel); @@ -429,14 +425,14 @@ bool boot_img::parse_image(const uint8_t *p, format_t type) { hdr->kernel_size() -= sizeof(mtk_hdr); k_fmt = check_fmt_lg(kernel, hdr->kernel_size()); } - if (k_fmt == ZIMAGE) { + if (k_fmt == FileFormat::ZIMAGE) { z_hdr = reinterpret_cast(kernel); const uint8_t* found_pos = 0; - + for (const uint8_t* search_pos = kernel + 0x28; search_pos < kernel + hdr->kernel_size(); search_pos++) { // ^^^^^^ +0x28 to search after zimage header and magic - if (check_fmt_lg(search_pos, hdr->kernel_size() - (search_pos - kernel)) != UNKNOWN) { + if (check_fmt_lg(search_pos, hdr->kernel_size() - (search_pos - kernel)) != FileFormat::UNKNOWN) { found_pos = search_pos; search_pos = kernel + hdr->kernel_size(); } @@ -488,7 +484,7 @@ bool boot_img::parse_image(const uint8_t *p, format_t type) { reinterpret_cast(vendor_ramdisk_table), hdr->vendor_ramdisk_table_entry_num()); for (auto &it : table) { - format_t fmt = check_fmt_lg(ramdisk + it.ramdisk_offset, it.ramdisk_size); + FileFormat fmt = check_fmt_lg(ramdisk + it.ramdisk_offset, it.ramdisk_size); fprintf(stderr, "%-*s name=[%s] type=[%s] size=[%u] fmt=[%s]\n", PADDING, "VND_RAMDISK", it.ramdisk_name, vendor_ramdisk_type(it.ramdisk_type), @@ -496,7 +492,7 @@ bool boot_img::parse_image(const uint8_t *p, format_t type) { } } else { r_fmt = check_fmt_lg(ramdisk, size); - if (r_fmt == MTK) { + if (r_fmt == FileFormat::MTK) { fprintf(stderr, "MTK_RAMDISK_HDR\n"); flags[MTK_RAMDISK] = true; r_hdr = reinterpret_cast(ramdisk); @@ -555,8 +551,8 @@ bool boot_img::verify(const char *cert) const { int split_image_dtb(const char *filename, bool skip_decomp) { mmap_data img(filename); - if (int off = find_dtb_offset(img.buf(), img.sz()); off > 0) { - format_t fmt = check_fmt_lg(img.buf(), img.sz()); + if (size_t off = find_dtb_offset(img.buf(), img.sz()); off > 0) { + FileFormat fmt = check_fmt_lg(img.buf(), img.sz()); if (!skip_decomp && COMPRESSED(fmt)) { int fd = creat(KERNEL_FILE, 0644); decompress(fmt, fd, img.buf(), off); @@ -609,7 +605,7 @@ int unpack(const char *image, bool skip_decomp, bool hdr) { ssprintf(file_name, sizeof(file_name), "%s.cpio", it.ramdisk_name); } owned_fd fd = xopenat(dirfd, file_name, O_CREAT | O_TRUNC | O_WRONLY | O_CLOEXEC, 0644); - format_t fmt = check_fmt_lg(boot.ramdisk + it.ramdisk_offset, it.ramdisk_size); + FileFormat fmt = check_fmt_lg(boot.ramdisk + it.ramdisk_offset, it.ramdisk_size); if (!skip_decomp && COMPRESSED(fmt)) { decompress(fmt, fd, boot.ramdisk + it.ramdisk_offset, it.ramdisk_size); } else { @@ -719,8 +715,8 @@ void repack(const char *src_img, const char *out_img, bool skip_comp) { mmap_data m(KERNEL_FILE); if (!skip_comp && !COMPRESSED_ANY(check_fmt(m.buf(), m.sz())) && COMPRESSED(boot.k_fmt)) { // Always use zopfli for zImage compression - auto fmt = (boot.flags[ZIMAGE_KERNEL] && boot.k_fmt == GZIP) ? ZOPFLI : boot.k_fmt; - hdr->kernel_size() = compress(fmt, fd, m.buf(), m.sz()); + auto fmt = (boot.flags[ZIMAGE_KERNEL] && boot.k_fmt == FileFormat::GZIP) ? FileFormat::ZOPFLI : boot.k_fmt; + hdr->kernel_size() = compress_len(fmt, m, fd); } else { hdr->kernel_size() = xwrite(fd, m.buf(), m.sz()); } @@ -783,10 +779,10 @@ void repack(const char *src_img, const char *out_img, bool skip_comp) { ssprintf(file_name, sizeof(file_name), "%s.cpio", it.ramdisk_name); } mmap_data m(dirfd, file_name); - format_t fmt = check_fmt_lg(boot.ramdisk + it.ramdisk_offset, it.ramdisk_size); + FileFormat fmt = check_fmt_lg(boot.ramdisk + it.ramdisk_offset, it.ramdisk_size); it.ramdisk_offset = ramdisk_offset; if (!skip_comp && !COMPRESSED_ANY(check_fmt(m.buf(), m.sz())) && COMPRESSED(fmt)) { - it.ramdisk_size = compress(fmt, fd, m.buf(), m.sz()); + it.ramdisk_size = compress_len(fmt, m, fd); } else { it.ramdisk_size = xwrite(fd, m.buf(), m.sz()); } @@ -798,15 +794,15 @@ void repack(const char *src_img, const char *out_img, bool skip_comp) { } else if (access(RAMDISK_FILE, R_OK) == 0) { mmap_data m(RAMDISK_FILE); auto r_fmt = boot.r_fmt; - if (!skip_comp && !hdr->is_vendor() && hdr->header_version() == 4 && r_fmt != LZ4_LEGACY) { + if (!skip_comp && !hdr->is_vendor() && hdr->header_version() == 4 && r_fmt != FileFormat::LZ4_LEGACY) { // A v4 boot image ramdisk will have to be merged with other vendor ramdisks, // and they have to use the exact same compression method. v4 GKIs are required to // use lz4 (legacy), so hardcode the format here. - fprintf(stderr, "RAMDISK_FMT: [%s] -> [%s]\n", fmt2name[r_fmt], fmt2name[LZ4_LEGACY]); - r_fmt = LZ4_LEGACY; + fprintf(stderr, "RAMDISK_FMT: [%s] -> [%s]\n", fmt2name[r_fmt], fmt2name[FileFormat::LZ4_LEGACY]); + r_fmt = FileFormat::LZ4_LEGACY; } if (!skip_comp && !COMPRESSED_ANY(check_fmt(m.buf(), m.sz())) && COMPRESSED(r_fmt)) { - hdr->ramdisk_size() = compress(r_fmt, fd, m.buf(), m.sz()); + hdr->ramdisk_size() = compress_len(r_fmt, m, fd); } else { hdr->ramdisk_size() = xwrite(fd, m.buf(), m.sz()); } @@ -825,7 +821,7 @@ void repack(const char *src_img, const char *out_img, bool skip_comp) { if (access(EXTRA_FILE, R_OK) == 0) { mmap_data m(EXTRA_FILE); if (!skip_comp && !COMPRESSED_ANY(check_fmt(m.buf(), m.sz())) && COMPRESSED(boot.e_fmt)) { - hdr->extra_size() = compress(boot.e_fmt, fd, m.buf(), m.sz()); + hdr->extra_size() = compress_len(boot.e_fmt, m, fd); } else { hdr->extra_size() = xwrite(fd, m.buf(), m.sz()); } diff --git a/native/src/boot/bootimg.hpp b/native/src/boot/bootimg.hpp index 2e185bc94..2e09f4afd 100644 --- a/native/src/boot/bootimg.hpp +++ b/native/src/boot/bootimg.hpp @@ -610,9 +610,9 @@ struct boot_img { std::bitset flags; // The format of kernel, ramdisk and extra - format_t k_fmt = UNKNOWN; - format_t r_fmt = UNKNOWN; - format_t e_fmt = UNKNOWN; + FileFormat k_fmt; + FileFormat r_fmt; + FileFormat e_fmt; /************************************************************* * Following pointers points within the read-only mmap region @@ -672,8 +672,8 @@ struct boot_img { boot_img(const char *); ~boot_img(); - bool parse_image(const uint8_t *addr, format_t type); - std::pair create_hdr(const uint8_t *addr, format_t type); + bool parse_image(const uint8_t *addr, FileFormat type); + std::pair create_hdr(const uint8_t *addr, FileFormat type); // Rust FFI rust::Slice get_payload() const { return payload; } diff --git a/native/src/boot/bzlib.h b/native/src/boot/bzlib.h deleted file mode 100644 index 96e1b546d..000000000 --- a/native/src/boot/bzlib.h +++ /dev/null @@ -1,287 +0,0 @@ - -/*-------------------------------------------------------------*/ -/*--- Public header file for the library. ---*/ -/*--- bzlib.h ---*/ -/*-------------------------------------------------------------*/ - -/* ------------------------------------------------------------------ - This file is part of bzip2/libbzip2, a program and library for - lossless, block-sorting data compression. - - bzip2/libbzip2 version 1.1.0 of 6 September 2010 - Copyright (C) 1996-2010 Julian Seward - - Please read the WARNING, DISCLAIMER and PATENTS sections in the - README file. - - This program is released under the terms of the license contained - in the file LICENSE. - ------------------------------------------------------------------ */ - - -#ifndef _BZLIB_H -#define _BZLIB_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define BZ_RUN 0 -#define BZ_FLUSH 1 -#define BZ_FINISH 2 - -#define BZ_OK 0 -#define BZ_RUN_OK 1 -#define BZ_FLUSH_OK 2 -#define BZ_FINISH_OK 3 -#define BZ_STREAM_END 4 -#define BZ_SEQUENCE_ERROR (-1) -#define BZ_PARAM_ERROR (-2) -#define BZ_MEM_ERROR (-3) -#define BZ_DATA_ERROR (-4) -#define BZ_DATA_ERROR_MAGIC (-5) -#define BZ_IO_ERROR (-6) -#define BZ_UNEXPECTED_EOF (-7) -#define BZ_OUTBUFF_FULL (-8) -#define BZ_CONFIG_ERROR (-9) - -typedef - struct { - char *next_in; - unsigned int avail_in; - unsigned int total_in_lo32; - unsigned int total_in_hi32; - - char *next_out; - unsigned int avail_out; - unsigned int total_out_lo32; - unsigned int total_out_hi32; - - void *state; - - void *(*bzalloc)(void *,int,int); - void (*bzfree)(void *,void *); - void *opaque; - } - bz_stream; - - -#ifndef BZ_IMPORT -#define BZ_EXPORT -#endif - -#ifndef BZ_NO_STDIO -/* Need a definitition for FILE */ -#include -#endif - -#ifdef _WIN32 -# include -# ifdef small - /* windows.h define small to char */ -# undef small -# endif -# ifndef WINAPI -# define WINAPI -# endif -# ifdef BZ_EXPORT -# define BZ_API(func) WINAPI func -# define BZ_EXTERN extern -# else - /* import windows dll dynamically */ -# define BZ_API(func) (WINAPI * func) -# define BZ_EXTERN -# endif -#else -# define BZ_API(func) func -#endif - -#ifndef BZ_EXTERN -#define BZ_EXTERN extern -#endif - -/*-- Core (low-level) library functions --*/ - -BZ_EXTERN int BZ_API(BZ2_bzCompressInit) ( - bz_stream* strm, - int blockSize100k, - int verbosity, - int workFactor - ); - -BZ_EXTERN int BZ_API(BZ2_bzCompress) ( - bz_stream* strm, - int action - ); - -BZ_EXTERN int BZ_API(BZ2_bzCompressEnd) ( - bz_stream* strm - ); - -BZ_EXTERN int BZ_API(BZ2_bzDecompressInit) ( - bz_stream *strm, - int verbosity, - int small - ); - -BZ_EXTERN int BZ_API(BZ2_bzDecompress) ( - bz_stream* strm - ); - -BZ_EXTERN int BZ_API(BZ2_bzDecompressEnd) ( - bz_stream *strm - ); - - - -/*-- High(er) level library functions --*/ - -#ifndef BZ_NO_STDIO -#define BZ_MAX_UNUSED 5000 - -typedef void BZFILE; - -BZ_EXTERN BZFILE* BZ_API(BZ2_bzReadOpen) ( - int* bzerror, - FILE* f, - int verbosity, - int small, - void* unused, - int nUnused - ); - -BZ_EXTERN void BZ_API(BZ2_bzReadClose) ( - int* bzerror, - BZFILE* b - ); - -BZ_EXTERN void BZ_API(BZ2_bzReadGetUnused) ( - int* bzerror, - BZFILE* b, - void** unused, - int* nUnused - ); - -BZ_EXTERN int BZ_API(BZ2_bzRead) ( - int* bzerror, - BZFILE* b, - void* buf, - int len - ); - -BZ_EXTERN BZFILE* BZ_API(BZ2_bzWriteOpen) ( - int* bzerror, - FILE* f, - int blockSize100k, - int verbosity, - int workFactor - ); - -BZ_EXTERN void BZ_API(BZ2_bzWrite) ( - int* bzerror, - BZFILE* b, - void* buf, - int len - ); - -BZ_EXTERN void BZ_API(BZ2_bzWriteClose) ( - int* bzerror, - BZFILE* b, - int abandon, - unsigned int* nbytes_in, - unsigned int* nbytes_out - ); - -BZ_EXTERN void BZ_API(BZ2_bzWriteClose64) ( - int* bzerror, - BZFILE* b, - int abandon, - unsigned int* nbytes_in_lo32, - unsigned int* nbytes_in_hi32, - unsigned int* nbytes_out_lo32, - unsigned int* nbytes_out_hi32 - ); -#endif - - -/*-- Utility functions --*/ - -BZ_EXTERN int BZ_API(BZ2_bzBuffToBuffCompress) ( - char* dest, - unsigned int* destLen, - char* source, - unsigned int sourceLen, - int blockSize100k, - int verbosity, - int workFactor - ); - -BZ_EXTERN int BZ_API(BZ2_bzBuffToBuffDecompress) ( - char* dest, - unsigned int* destLen, - char* source, - unsigned int sourceLen, - int small, - int verbosity - ); - - -/*-- - Code contributed by Yoshioka Tsuneo (tsuneo@rr.iij4u.or.jp) - to support better zlib compatibility. - This code is not _officially_ part of libbzip2 (yet); - I haven't tested it, documented it, or considered the - threading-safeness of it. - If this code breaks, please contact both Yoshioka and me. ---*/ - -BZ_EXTERN const char * BZ_API(BZ2_bzlibVersion) ( - void - ); - -#ifndef BZ_NO_STDIO -BZ_EXTERN BZFILE * BZ_API(BZ2_bzopen) ( - const char *path, - const char *mode - ); - -BZ_EXTERN BZFILE * BZ_API(BZ2_bzdopen) ( - int fd, - const char *mode - ); - -BZ_EXTERN int BZ_API(BZ2_bzread) ( - BZFILE* b, - void* buf, - int len - ); - -BZ_EXTERN int BZ_API(BZ2_bzwrite) ( - BZFILE* b, - void* buf, - int len - ); - -BZ_EXTERN int BZ_API(BZ2_bzflush) ( - BZFILE* b - ); - -BZ_EXTERN void BZ_API(BZ2_bzclose) ( - BZFILE* b - ); - -BZ_EXTERN const char * BZ_API(BZ2_bzerror) ( - BZFILE *b, - int *errnum - ); -#endif - -#ifdef __cplusplus -} -#endif - -#endif - -/*-------------------------------------------------------------*/ -/*--- end bzlib.h ---*/ -/*-------------------------------------------------------------*/ diff --git a/native/src/boot/compress.cpp b/native/src/boot/compress.cpp deleted file mode 100644 index c99a8aa3c..000000000 --- a/native/src/boot/compress.cpp +++ /dev/null @@ -1,762 +0,0 @@ -#include -#include - -#include -#include "bzlib.h" -#include -#include -#include -#include -#include -#include - -#include - -#include "magiskboot.hpp" -#include "compress.hpp" - -using namespace std; - -#define bwrite this->base->write - -constexpr size_t CHUNK = 0x40000; -constexpr size_t LZ4_UNCOMPRESSED = 0x800000; -constexpr size_t LZ4_COMPRESSED = LZ4_COMPRESSBOUND(LZ4_UNCOMPRESSED); - -class gz_strm : public filter_out_stream { -public: - bool write(const void *buf, size_t len) override { - return len == 0 || do_write(buf, len, Z_NO_FLUSH); - } - - ~gz_strm() override { - do_write(nullptr, 0, Z_FINISH); - switch(mode) { - case DECODE: - inflateEnd(&strm); - break; - case ENCODE: - deflateEnd(&strm); - break; - default: - break; - } - } - -protected: - enum mode_t { - DECODE, - ENCODE, - WAIT, - COPY - } mode; - - gz_strm(mode_t mode, out_strm_ptr &&base) : - filter_out_stream(std::move(base)), mode(mode), strm{}, outbuf{0} { - switch(mode) { - case DECODE: - inflateInit2(&strm, 15 | 16); - break; - case ENCODE: - deflateInit2(&strm, 9, Z_DEFLATED, 15 | 16, 8, Z_DEFAULT_STRATEGY); - break; - default: - break; - } - } - -private: - z_stream strm; - uint8_t outbuf[CHUNK]; - - bool do_write(const void *buf, size_t len, int flush) { - if (mode == WAIT) { - if (len == 0) return true; - Bytef b[1] = {0x1f}; - if (*(Bytef *)buf == 0x8b) { - mode = DECODE; - inflateReset(&strm); - strm.next_in = b; - strm.avail_in = 1; - inflate(&strm, flush); - } else { - mode = COPY; - return true; - } - } - strm.next_in = (Bytef *) buf; - strm.avail_in = len; - do { - int code; - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - switch(mode) { - case DECODE: - code = inflate(&strm, flush); - break; - case ENCODE: - code = deflate(&strm, flush); - break; - case COPY: - return true; - default: - // should have been handled - return false; - } - if (code == Z_STREAM_ERROR) { - LOGW("gzip %s failed (%d)\n", mode ? "encode" : "decode", code); - return false; - } - if (!bwrite(outbuf, sizeof(outbuf) - strm.avail_out)) - return false; - if (mode == DECODE && code == Z_STREAM_END) { - if (strm.avail_in > 1) { - if (strm.next_in[0] == 0x1f && strm.next_in[1] == 0x8b) { - // There is still data in the stream, we need to reset the stream - // and continue decoding - inflateReset(&strm); - strm.avail_out = 0; - continue; - } - } else if (strm.avail_in == 1) { - if (strm.next_in[0] == 0x1f) { - // If there is only one byte left, we need to wait for the next byte - // to determine if it is a gzip header - mode = WAIT; - return true; - } - } else { - // The next inflate won't consume any data but fallback - // to the previous two conditions - return true; - } - // There is still data in the stream, we need to copy it - mode = COPY; - return true; - } - } while (strm.avail_out == 0); - return true; - } -}; - -class gz_decoder : public gz_strm { -public: - explicit gz_decoder(out_strm_ptr &&base) : gz_strm(DECODE, std::move(base)) {}; -}; - -class gz_encoder : public gz_strm { -public: - explicit gz_encoder(out_strm_ptr &&base) : gz_strm(ENCODE, std::move(base)) {}; -}; - -class zopfli_encoder : public chunk_out_stream { -public: - explicit zopfli_encoder(out_strm_ptr &&base) : - chunk_out_stream(std::move(base), ZOPFLI_MASTER_BLOCK_SIZE), - zo{}, out(nullptr), outsize(0), crc(crc32(0L, Z_NULL, 0)), in_total(0), bp(0) { - ZopfliInitOptions(&zo); - - // This config is already better than gzip -9 - zo.numiterations = 1; - zo.blocksplitting = 0; - - ZOPFLI_APPEND_DATA(31, &out, &outsize); /* ID1 */ - ZOPFLI_APPEND_DATA(139, &out, &outsize); /* ID2 */ - ZOPFLI_APPEND_DATA(8, &out, &outsize); /* CM */ - ZOPFLI_APPEND_DATA(0, &out, &outsize); /* FLG */ - /* MTIME */ - ZOPFLI_APPEND_DATA(0, &out, &outsize); - ZOPFLI_APPEND_DATA(0, &out, &outsize); - ZOPFLI_APPEND_DATA(0, &out, &outsize); - ZOPFLI_APPEND_DATA(0, &out, &outsize); - - ZOPFLI_APPEND_DATA(2, &out, &outsize); /* XFL, 2 indicates best compression. */ - ZOPFLI_APPEND_DATA(3, &out, &outsize); /* OS follows Unix conventions. */ - } - - ~zopfli_encoder() override { - finalize(); - - /* CRC */ - ZOPFLI_APPEND_DATA(crc % 256, &out, &outsize); - ZOPFLI_APPEND_DATA((crc >> 8) % 256, &out, &outsize); - ZOPFLI_APPEND_DATA((crc >> 16) % 256, &out, &outsize); - ZOPFLI_APPEND_DATA((crc >> 24) % 256, &out, &outsize); - - /* ISIZE */ - ZOPFLI_APPEND_DATA(in_total % 256, &out, &outsize); - ZOPFLI_APPEND_DATA((in_total >> 8) % 256, &out, &outsize); - ZOPFLI_APPEND_DATA((in_total >> 16) % 256, &out, &outsize); - ZOPFLI_APPEND_DATA((in_total >> 24) % 256, &out, &outsize); - - bwrite(out, outsize); - free(out); - } - -protected: - bool write_chunk(const void *buf, size_t len, bool final) override { - auto in = static_cast(buf); - - in_total += len; - crc = crc32(crc, in, len); - - ZopfliDeflatePart(&zo, 2, final, in, 0, len, &bp, &out, &outsize); - - // ZOPFLI_APPEND_DATA is extremely dumb, so we always preserve the - // last byte to make sure that realloc is used instead of malloc - if (!bwrite(out, outsize - 1)) - return false; - out[0] = out[outsize - 1]; - outsize = 1; - - return true; - } - -private: - ZopfliOptions zo; - unsigned char *out; - size_t outsize; - unsigned long crc; - uint32_t in_total; - unsigned char bp; -}; - -class bz_strm : public filter_out_stream { -public: - bool write(const void *buf, size_t len) override { - return len == 0 || do_write(buf, len, BZ_RUN); - } - - ~bz_strm() override { - switch(mode) { - case DECODE: - BZ2_bzDecompressEnd(&strm); - break; - case ENCODE: - do_write(nullptr, 0, BZ_FINISH); - BZ2_bzCompressEnd(&strm); - break; - } - } - -protected: - enum mode_t { - DECODE, - ENCODE - } mode; - - bz_strm(mode_t mode, out_strm_ptr &&base) : - filter_out_stream(std::move(base)), mode(mode), strm{}, outbuf{0} { - switch(mode) { - case DECODE: - BZ2_bzDecompressInit(&strm, 0, 0); - break; - case ENCODE: - BZ2_bzCompressInit(&strm, 9, 0, 0); - break; - } - } - -private: - bz_stream strm; - char outbuf[CHUNK]; - - bool do_write(const void *buf, size_t len, int flush) { - strm.next_in = (char *) buf; - strm.avail_in = len; - do { - int code; - strm.avail_out = sizeof(outbuf); - strm.next_out = outbuf; - switch(mode) { - case DECODE: - code = BZ2_bzDecompress(&strm); - break; - case ENCODE: - code = BZ2_bzCompress(&strm, flush); - break; - } - if (code < 0) { - LOGW("bzip2 %s failed (%d)\n", mode ? "encode" : "decode", code); - return false; - } - if (!bwrite(outbuf, sizeof(outbuf) - strm.avail_out)) - return false; - if (code == BZ_STREAM_END) - return true; - } while (strm.avail_out == 0); - return true; - } -}; - -class bz_decoder : public bz_strm { -public: - explicit bz_decoder(out_strm_ptr &&base) : bz_strm(DECODE, std::move(base)) {}; -}; - -class bz_encoder : public bz_strm { -public: - explicit bz_encoder(out_strm_ptr &&base) : bz_strm(ENCODE, std::move(base)) {}; -}; - -class lzma_strm : public filter_out_stream { -public: - bool write(const void *buf, size_t len) override { - return len == 0 || do_write(buf, len, LZMA_RUN); - } - - ~lzma_strm() override { - do_write(nullptr, 0, LZMA_FINISH); - lzma_end(&strm); - } - -protected: - enum mode_t { - DECODE, - ENCODE_XZ, - ENCODE_LZMA - } mode; - - lzma_strm(mode_t mode, out_strm_ptr &&base) : - filter_out_stream(std::move(base)), mode(mode), strm(LZMA_STREAM_INIT), outbuf{0} { - lzma_options_lzma opt; - - // Initialize preset - lzma_lzma_preset(&opt, 9); - lzma_filter filters[] = { - { .id = LZMA_FILTER_LZMA2, .options = &opt }, - { .id = LZMA_VLI_UNKNOWN, .options = nullptr }, - }; - - lzma_ret code; - switch(mode) { - case DECODE: - code = lzma_auto_decoder(&strm, UINT64_MAX, 0); - break; - case ENCODE_XZ: - code = lzma_stream_encoder(&strm, filters, LZMA_CHECK_CRC32); - break; - case ENCODE_LZMA: - code = lzma_alone_encoder(&strm, &opt); - break; - } - if (code != LZMA_OK) { - LOGE("LZMA initialization failed (%d)\n", code); - } - } - -private: - lzma_stream strm; - uint8_t outbuf[CHUNK]; - - bool do_write(const void *buf, size_t len, lzma_action flush) { - strm.next_in = (uint8_t *) buf; - strm.avail_in = len; - do { - strm.avail_out = sizeof(outbuf); - strm.next_out = outbuf; - int code = lzma_code(&strm, flush); - if (code != LZMA_OK && code != LZMA_STREAM_END) { - LOGW("LZMA %s failed (%d)\n", mode ? "encode" : "decode", code); - return false; - } - if (!bwrite(outbuf, sizeof(outbuf) - strm.avail_out)) - return false; - } while (strm.avail_out == 0); - return true; - } -}; - -class lzma_decoder : public lzma_strm { -public: - explicit lzma_decoder(out_strm_ptr &&base) : lzma_strm(DECODE, std::move(base)) {} -}; - -class xz_encoder : public lzma_strm { -public: - explicit xz_encoder(out_strm_ptr &&base) : lzma_strm(ENCODE_XZ, std::move(base)) {} -}; - -class lzma_encoder : public lzma_strm { -public: - explicit lzma_encoder(out_strm_ptr &&base) : lzma_strm(ENCODE_LZMA, std::move(base)) {} -}; - -class LZ4F_decoder : public filter_out_stream { -public: - explicit LZ4F_decoder(out_strm_ptr &&base) : - filter_out_stream(std::move(base)), ctx(nullptr), outbuf(nullptr), outCapacity(0) { - LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); - } - - ~LZ4F_decoder() override { - LZ4F_freeDecompressionContext(ctx); - delete[] outbuf; - } - - bool write(const void *buf, size_t len) override { - auto in = reinterpret_cast(buf); - if (!outbuf) { - size_t read = len; - LZ4F_frameInfo_t info; - LZ4F_getFrameInfo(ctx, &info, in, &read); - switch (info.blockSizeID) { - case LZ4F_default: - case LZ4F_max64KB: outCapacity = 1 << 16; break; - case LZ4F_max256KB: outCapacity = 1 << 18; break; - case LZ4F_max1MB: outCapacity = 1 << 20; break; - case LZ4F_max4MB: outCapacity = 1 << 22; break; - } - outbuf = new uint8_t[outCapacity]; - in += read; - len -= read; - } - size_t read, write; - LZ4F_errorCode_t code; - do { - read = len; - write = outCapacity; - code = LZ4F_decompress(ctx, outbuf, &write, in, &read, nullptr); - if (LZ4F_isError(code)) { - LOGW("LZ4F decode error: %s\n", LZ4F_getErrorName(code)); - return false; - } - len -= read; - in += read; - if (!bwrite(outbuf, write)) - return false; - } while (len != 0 || write != 0); - return true; - } - -private: - LZ4F_decompressionContext_t ctx; - uint8_t *outbuf; - size_t outCapacity; -}; - -class LZ4F_encoder : public filter_out_stream { -public: - explicit LZ4F_encoder(out_strm_ptr &&base) : - filter_out_stream(std::move(base)), ctx(nullptr), out_buf(nullptr), outCapacity(0) { - LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); - } - - bool write(const void *buf, size_t len) override { - if (!out_buf) { - LZ4F_preferences_t prefs { - .frameInfo = { - .blockSizeID = LZ4F_max4MB, - .blockMode = LZ4F_blockIndependent, - .contentChecksumFlag = LZ4F_contentChecksumEnabled, - .blockChecksumFlag = LZ4F_noBlockChecksum, - }, - .compressionLevel = 9, - .autoFlush = 1, - }; - outCapacity = LZ4F_compressBound(BLOCK_SZ, &prefs); - out_buf = new uint8_t[outCapacity]; - size_t write = LZ4F_compressBegin(ctx, out_buf, outCapacity, &prefs); - if (!bwrite(out_buf, write)) - return false; - } - if (len == 0) - return true; - - auto in = reinterpret_cast(buf); - size_t read, write; - do { - read = len > BLOCK_SZ ? BLOCK_SZ : len; - write = LZ4F_compressUpdate(ctx, out_buf, outCapacity, in, read, nullptr); - if (LZ4F_isError(write)) { - LOGW("LZ4F encode error: %s\n", LZ4F_getErrorName(write)); - return false; - } - len -= read; - in += read; - if (!bwrite(out_buf, write)) - return false; - } while (len != 0); - return true; - } - - ~LZ4F_encoder() override { - size_t len = LZ4F_compressEnd(ctx, out_buf, outCapacity, nullptr); - if (LZ4F_isError(len)) { - LOGE("LZ4F end of frame error: %s\n", LZ4F_getErrorName(len)); - } else if (!bwrite(out_buf, len)) { - LOGE("LZ4F end of frame error: I/O error\n"); - } - LZ4F_freeCompressionContext(ctx); - delete[] out_buf; - } - -private: - LZ4F_compressionContext_t ctx; - uint8_t *out_buf; - size_t outCapacity; - - static constexpr size_t BLOCK_SZ = 1 << 22; -}; - -class LZ4_decoder : public chunk_out_stream { -public: - explicit LZ4_decoder(out_strm_ptr &&base) : - chunk_out_stream(std::move(base), LZ4_COMPRESSED, sizeof(block_sz)), - out_buf(new char[LZ4_UNCOMPRESSED]), block_sz(0) {} - - ~LZ4_decoder() override { - finalize(); - delete[] out_buf; - } - -protected: - bool write_chunk(const void *buf, size_t len, bool) override { - // This is an error - if (len != chunk_sz) - return false; - - auto in = reinterpret_cast(buf); - - if (block_sz == 0) { - memcpy(&block_sz, in, sizeof(block_sz)); - if (block_sz == 0x184C2102) { - // This is actually the lz4 magic, read the next 4 bytes - block_sz = 0; - chunk_sz = sizeof(block_sz); - return true; - } - // Read the next block chunk - chunk_sz = block_sz; - return true; - } else { - int r = LZ4_decompress_safe(in, out_buf, block_sz, LZ4_UNCOMPRESSED); - chunk_sz = sizeof(block_sz); - block_sz = 0; - if (r < 0) { - LOGW("LZ4HC decompression failure (%d)\n", r); - return false; - } - return bwrite(out_buf, r); - } - } - -private: - char *out_buf; - uint32_t block_sz; -}; - -class LZ4_encoder : public chunk_out_stream { -public: - explicit LZ4_encoder(out_strm_ptr &&base, bool lg) : - chunk_out_stream(std::move(base), LZ4_UNCOMPRESSED), - out_buf(new char[LZ4_COMPRESSED]), lg(lg), in_total(0) { - bwrite("\x02\x21\x4c\x18", 4); - } - - ~LZ4_encoder() override { - finalize(); - if (lg) - bwrite(&in_total, sizeof(in_total)); - delete[] out_buf; - } - -protected: - bool write_chunk(const void *buf, size_t len, bool) override { - auto in = static_cast(buf); - uint32_t block_sz = LZ4_compress_HC(in, out_buf, len, LZ4_COMPRESSED, LZ4HC_CLEVEL_MAX); - if (block_sz == 0) { - LOGW("LZ4HC compression failure\n"); - return false; - } - if (bwrite(&block_sz, sizeof(block_sz)) && bwrite(out_buf, block_sz)) { - in_total += len; - return true; - } - return false; - } - -private: - char *out_buf; - bool lg; - uint32_t in_total; -}; - -out_strm_ptr get_encoder(format_t type, out_strm_ptr &&base) { - switch (type) { - case XZ: - return make_unique(std::move(base)); - case LZMA: - return make_unique(std::move(base)); - case BZIP2: - return make_unique(std::move(base)); - case LZ4: - return make_unique(std::move(base)); - case LZ4_LEGACY: - return make_unique(std::move(base), false); - case LZ4_LG: - return make_unique(std::move(base), true); - case ZOPFLI: - return make_unique(std::move(base)); - case GZIP: - default: - return make_unique(std::move(base)); - } -} - -out_strm_ptr get_decoder(format_t type, out_strm_ptr &&base) { - switch (type) { - case XZ: - case LZMA: - return make_unique(std::move(base)); - case BZIP2: - return make_unique(std::move(base)); - case LZ4: - return make_unique(std::move(base)); - case LZ4_LEGACY: - case LZ4_LG: - return make_unique(std::move(base)); - case ZOPFLI: - case GZIP: - default: - return make_unique(std::move(base)); - } -} - -void decompress(char *infile, const char *outfile) { - bool in_std = infile == "-"sv; - bool rm_in = false; - - int in_fd = in_std ? STDIN_FILENO : xopen(infile, O_RDONLY); - int out_fd = -1; - out_strm_ptr strm; - - char buf[4096]; - size_t len; - while ((len = read(in_fd, buf, sizeof(buf)))) { - if (!strm) { - format_t type = check_fmt(buf, len); - - fprintf(stderr, "Detected format: [%s]\n", fmt2name[type]); - - if (!COMPRESSED(type)) - LOGE("Input file is not a supported compressed type!\n"); - - /* If user does not provide outfile, infile has to be either - * .[ext], or '-'. Outfile will be either or '-'. - * If the input does not have proper format, abort */ - - char *ext = nullptr; - if (outfile == nullptr) { - outfile = infile; - if (!in_std) { - ext = strrchr(infile, '.'); - if (ext == nullptr || strcmp(ext, fmt2ext[type]) != 0) - LOGE("Input file is not a supported type!\n"); - - // Strip out extension and remove input - *ext = '\0'; - rm_in = true; - fprintf(stderr, "Decompressing to [%s]\n", outfile); - } - } - - out_fd = outfile == "-"sv ? - STDOUT_FILENO : - xopen(outfile, O_WRONLY | O_CREAT | O_TRUNC, 0644); - strm = get_decoder(type, make_unique(out_fd)); - if (ext) *ext = '.'; - } - if (!strm->write(buf, len)) - LOGE("Decompression error!\n"); - } - - strm.reset(nullptr); - if (in_fd != STDIN_FILENO) close(in_fd); - if (out_fd != STDOUT_FILENO) close(out_fd); - - if (rm_in) - unlink(infile); -} - -void compress(const char *method, const char *infile, const char *outfile) { - format_t fmt = name2fmt[method]; - if (fmt == UNKNOWN) - LOGE("Unknown compression method: [%s]\n", method); - - bool in_std = infile == "-"sv; - bool rm_in = false; - - int in_fd = in_std ? STDIN_FILENO : xopen(infile, O_RDONLY); - int out_fd = -1; - - if (outfile == nullptr) { - if (in_std) { - out_fd = STDOUT_FILENO; - } else { - /* If user does not provide outfile and infile is not - * STDIN, output to .[ext] */ - string tmp(infile); - tmp += fmt2ext[fmt]; - out_fd = xopen(tmp.data(), O_WRONLY | O_CREAT | O_TRUNC, 0644); - fprintf(stderr, "Compressing to [%s]\n", tmp.data()); - rm_in = true; - } - } else { - out_fd = outfile == "-"sv ? - STDOUT_FILENO : - xopen(outfile, O_WRONLY | O_CREAT | O_TRUNC, 0644); - } - - auto strm = get_encoder(fmt, make_unique(out_fd)); - - char buf[4096]; - size_t len; - while ((len = read(in_fd, buf, sizeof(buf)))) { - if (!strm->write(buf, len)) - LOGE("Compression error!\n"); - } - - strm.reset(nullptr); - if (in_fd != STDIN_FILENO) close(in_fd); - if (out_fd != STDOUT_FILENO) close(out_fd); - - if (rm_in) - unlink(infile); -} - -bool decompress(rust::Slice buf, int fd) { - format_t type = check_fmt(buf.data(), buf.length()); - - if (!COMPRESSED(type)) { - LOGE("Input file is not a supported compression format!\n"); - return false; - } - - auto strm = get_decoder(type, make_unique(fd)); - if (!strm->write(buf.data(), buf.length())) { - return false; - } - return true; -} - -bool xz(rust::Slice buf, rust::Vec &out) { - auto strm = get_encoder(XZ, make_unique(out)); - if (!strm->write(buf.data(), buf.length())) { - return false; - } - return true; -} - -bool unxz(rust::Slice buf, rust::Vec &out) { - format_t type = check_fmt(buf.data(), buf.length()); - if (type != XZ) { - LOGE("Input file is not in xz format!\n"); - return false; - } - auto strm = get_decoder(XZ, make_unique(out)); - if (!strm->write(buf.data(), buf.length())) { - return false; - } - return true; -} diff --git a/native/src/boot/compress.hpp b/native/src/boot/compress.hpp deleted file mode 100644 index bcb32e59f..000000000 --- a/native/src/boot/compress.hpp +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include -#include - -#include "format.hpp" - -out_strm_ptr get_encoder(format_t type, out_strm_ptr &&base); -out_strm_ptr get_decoder(format_t type, out_strm_ptr &&base); -void compress(const char *method, const char *infile, const char *outfile); -void decompress(char *infile, const char *outfile); -bool decompress(rust::Slice buf, int fd); -bool xz(rust::Slice buf, rust::Vec &out); -bool unxz(rust::Slice buf, rust::Vec &out); diff --git a/native/src/boot/compress.rs b/native/src/boot/compress.rs new file mode 100644 index 000000000..597154c04 --- /dev/null +++ b/native/src/boot/compress.rs @@ -0,0 +1,426 @@ +use crate::ffi::FileFormat; +use base::{Chunker, LoggedResult, WriteExt}; +use bytemuck::bytes_of_mut; +use bzip2::{Compression as BzCompression, write::BzDecoder, write::BzEncoder}; +use flate2::{Compression as GzCompression, write::GzEncoder, write::MultiGzDecoder}; +use lz4::{ + BlockMode, BlockSize, ContentChecksum, Encoder as LZ4FrameEncoder, + EncoderBuilder as LZ4FrameEncoderBuilder, block::CompressionMode, liblz4::BlockChecksum, +}; +use std::cell::Cell; +use std::fs::File; +use std::io::{BufWriter, Read, Write}; +use std::mem::ManuallyDrop; +use std::num::NonZeroU64; +use std::ops::DerefMut; +use std::os::fd::{FromRawFd, RawFd}; +use xz2::{ + stream::{Check as LzmaCheck, Filters as LzmaFilters, LzmaOptions, Stream as LzmaStream}, + write::{XzDecoder, XzEncoder}, +}; +use zopfli::{BlockType, GzipEncoder as ZopFliEncoder, Options as ZopfliOptions}; + +pub trait WriteFinish: Write { + fn finish(self: Box) -> std::io::Result; +} + +// Boilerplate for existing types + +macro_rules! finish_impl { + ($($t:ty),*) => {$( + impl WriteFinish for $t { + fn finish(self: Box) -> std::io::Result { + Self::finish(*self) + } + } + )*} +} + +finish_impl!(GzEncoder, MultiGzDecoder, BzEncoder, XzEncoder); + +macro_rules! finish_impl_ref { + ($($t:ty),*) => {$( + impl WriteFinish for $t { + fn finish(mut self: Box) -> std::io::Result { + Self::finish(self.as_mut()) + } + } + )*} +} + +finish_impl_ref!(BzDecoder, XzDecoder); + +impl WriteFinish for BufWriter> { + fn finish(self: Box) -> std::io::Result { + let inner = self.into_inner()?; + ZopFliEncoder::finish(inner) + } +} + +impl WriteFinish for LZ4FrameEncoder { + fn finish(self: Box) -> std::io::Result { + let (w, r) = Self::finish(*self); + r?; + Ok(w) + } +} + +// Adapt Reader to Writer + +// In case some decoders don't support the Write trait, instead of pushing data into the +// decoder, we have no choice but to pull data out of it. So first, we create a "fake" reader +// that does not own any data as a placeholder. In the Writer adapter struct, when data +// is fed in, we call FakeReader::set_data to forward this data as the "source" of the +// decoder. Next, we pull data out of the decoder, and finally, forward the decoded data to output. + +struct FakeReader(Cell<&'static [u8]>); + +impl FakeReader { + fn new() -> FakeReader { + FakeReader(Cell::new(&[])) + } + + // SAFETY: the lifetime of the buffer is between the invocation of + // this method and the invocation of FakeReader::clear. There is currently + // no way to represent this with Rust's lifetime marker, so we transmute all + // lifetimes away and make the users of this struct manually manage the lifetime. + // It is the responsibility of the caller to ensure the underlying reference does not + // live longer than it should. + unsafe fn set_data(&self, data: &[u8]) { + let buf: &'static [u8] = unsafe { std::mem::transmute(data) }; + self.0.set(buf) + } + + fn clear(&self) { + self.0.set(&[]) + } +} + +impl Read for FakeReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let data = self.0.get(); + let len = std::cmp::min(buf.len(), data.len()); + buf[..len].copy_from_slice(&data[..len]); + self.0.set(&data[len..]); + Ok(len) + } +} + +// LZ4FrameDecoder + +struct LZ4FrameDecoder { + write: W, + decoder: lz4::Decoder, +} + +impl LZ4FrameDecoder { + fn new(write: W) -> Self { + let fake = FakeReader::new(); + let decoder = lz4::Decoder::new(fake).unwrap(); + LZ4FrameDecoder { write, decoder } + } +} + +impl Write for LZ4FrameDecoder { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.write_all(buf)?; + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + + fn write_all(&mut self, buf: &[u8]) -> std::io::Result<()> { + // SAFETY: buf is removed from the reader immediately after usage + unsafe { self.decoder.reader().set_data(buf) }; + std::io::copy(&mut self.decoder, &mut self.write)?; + self.decoder.reader().clear(); + Ok(()) + } +} + +impl WriteFinish for LZ4FrameDecoder { + fn finish(self: Box) -> std::io::Result { + let (_, r) = self.decoder.finish(); + r?; + Ok(self.write) + } +} + +// LZ4BlockArchive format +// +// len: | 4 | 4 | n | ... | 4 | +// data: | magic | compressed block size | compressed block data | ... | total uncompressed size | + +// LZ4BlockEncoder + +const LZ4_BLOCK_SIZE: usize = 0x800000; +const LZ4HC_CLEVEL_MAX: i32 = 12; +const LZ4_MAGIC: &[u8] = b"\x02\x21\x4c\x18"; + +struct LZ4BlockEncoder { + write: W, + chunker: Chunker, + out_buf: Box<[u8]>, + total: u32, + is_lg: bool, +} + +impl LZ4BlockEncoder { + fn new(write: W, is_lg: bool) -> Self { + let out_sz = lz4::block::compress_bound(LZ4_BLOCK_SIZE).unwrap_or(LZ4_BLOCK_SIZE); + LZ4BlockEncoder { + write, + chunker: Chunker::new(LZ4_BLOCK_SIZE), + // SAFETY: all bytes will be initialized before it is used + out_buf: unsafe { Box::new_uninit_slice(out_sz).assume_init() }, + total: 0, + is_lg, + } + } + + fn encode_block(write: &mut W, out_buf: &mut [u8], chunk: &[u8]) -> std::io::Result<()> { + let compressed_size = lz4::block::compress_to_buffer( + chunk, + Some(CompressionMode::HIGHCOMPRESSION(LZ4HC_CLEVEL_MAX)), + false, + out_buf, + )?; + let block_size = compressed_size as u32; + write.write_pod(&block_size)?; + write.write_all(&out_buf[..compressed_size]) + } +} + +impl Write for LZ4BlockEncoder { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.write_all(buf)?; + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + + fn write_all(&mut self, mut buf: &[u8]) -> std::io::Result<()> { + if self.total == 0 { + // Write header + self.write.write_all(LZ4_MAGIC)?; + } + + self.total += buf.len() as u32; + while !buf.is_empty() { + let (b, chunk) = self.chunker.add_data(buf); + buf = b; + if let Some(chunk) = chunk { + Self::encode_block(&mut self.write, &mut self.out_buf, chunk)?; + } + } + Ok(()) + } +} + +impl WriteFinish for LZ4BlockEncoder { + fn finish(mut self: Box) -> std::io::Result { + let chunk = self.chunker.get_available(); + if !chunk.is_empty() { + Self::encode_block(&mut self.write, &mut self.out_buf, chunk)?; + } + if self.is_lg { + self.write.write_pod(&self.total)?; + } + Ok(self.write) + } +} + +// LZ4BlockDecoder + +struct LZ4BlockDecoder { + write: W, + chunker: Chunker, + out_buf: Box<[u8]>, + curr_block_size: usize, +} + +impl LZ4BlockDecoder { + fn new(write: W) -> Self { + LZ4BlockDecoder { + write, + chunker: Chunker::new(size_of::()), + // SAFETY: all bytes will be initialized before it is used + out_buf: unsafe { Box::new_uninit_slice(LZ4_BLOCK_SIZE).assume_init() }, + curr_block_size: 0, + } + } + + fn decode_block(write: &mut W, out_buf: &mut [u8], chunk: &[u8]) -> std::io::Result<()> { + let decompressed_size = + lz4::block::decompress_to_buffer(chunk, Some(LZ4_BLOCK_SIZE as i32), out_buf)?; + write.write_all(&out_buf[..decompressed_size]) + } +} + +impl Write for LZ4BlockDecoder { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.write_all(buf)?; + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + + fn write_all(&mut self, mut buf: &[u8]) -> std::io::Result<()> { + while !buf.is_empty() { + let (b, chunk) = self.chunker.add_data(buf); + buf = b; + if let Some(chunk) = chunk { + if chunk == LZ4_MAGIC { + // Skip magic, read next u32 + continue; + } + if self.curr_block_size == 0 { + // We haven't got the current block size yet, try read it + let mut next_u32: u32 = 0; + bytes_of_mut(&mut next_u32).copy_from_slice(chunk); + + if next_u32 > lz4::block::compress_bound(LZ4_BLOCK_SIZE)? as u32 { + // This is the LG format trailer, EOF + continue; + } + + // Update chunker to read next block + self.curr_block_size = next_u32 as usize; + self.chunker.set_chunk_size(self.curr_block_size); + continue; + } + + // Actually decode the block + Self::decode_block(&mut self.write, &mut self.out_buf, chunk)?; + + // Reset for the next block + self.curr_block_size = 0; + self.chunker.set_chunk_size(size_of::()); + } + } + Ok(()) + } +} + +impl WriteFinish for LZ4BlockDecoder { + fn finish(mut self: Box) -> std::io::Result { + let chunk = self.chunker.get_available(); + if !chunk.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::Interrupted, + "Finish ran before end of compressed stream", + )); + } + Ok(self.write) + } +} + +// Top-level APIs + +pub fn get_encoder<'a, W: Write + 'a>(format: FileFormat, w: W) -> Box + 'a> { + match format { + FileFormat::XZ => { + let opt = LzmaOptions::new_preset(9).unwrap(); + let stream = + LzmaStream::new_stream_encoder(LzmaFilters::new().lzma2(&opt), LzmaCheck::Crc32) + .unwrap(); + Box::new(XzEncoder::new_stream(w, stream)) + } + FileFormat::LZMA => { + let opt = LzmaOptions::new_preset(9).unwrap(); + let stream = LzmaStream::new_lzma_encoder(&opt).unwrap(); + Box::new(XzEncoder::new_stream(w, stream)) + } + FileFormat::BZIP2 => Box::new(BzEncoder::new(w, BzCompression::best())), + FileFormat::LZ4 => { + let encoder = LZ4FrameEncoderBuilder::new() + .block_size(BlockSize::Max4MB) + .block_mode(BlockMode::Independent) + .checksum(ContentChecksum::ChecksumEnabled) + .block_checksum(BlockChecksum::BlockChecksumEnabled) + .level(9) + .auto_flush(true) + .build(w) + .unwrap(); + Box::new(encoder) + } + FileFormat::LZ4_LEGACY => Box::new(LZ4BlockEncoder::new(w, false)), + FileFormat::LZ4_LG => Box::new(LZ4BlockEncoder::new(w, true)), + FileFormat::ZOPFLI => { + // These options are already better than gzip -9 + let opt = ZopfliOptions { + iteration_count: NonZeroU64::new(1).unwrap(), + maximum_block_splits: 1, + ..Default::default() + }; + Box::new(ZopFliEncoder::new_buffered(opt, BlockType::Dynamic, w).unwrap()) + } + FileFormat::GZIP => Box::new(GzEncoder::new(w, GzCompression::best())), + _ => unreachable!(), + } +} + +pub fn get_decoder<'a, W: Write + 'a>(format: FileFormat, w: W) -> Box + 'a> { + match format { + FileFormat::XZ | FileFormat::LZMA => { + let stream = LzmaStream::new_auto_decoder(u64::MAX, 0).unwrap(); + Box::new(XzDecoder::new_stream(w, stream)) + } + FileFormat::BZIP2 => Box::new(BzDecoder::new(w)), + FileFormat::LZ4 => Box::new(LZ4FrameDecoder::new(w)), + FileFormat::LZ4_LG | FileFormat::LZ4_LEGACY => Box::new(LZ4BlockDecoder::new(w)), + FileFormat::ZOPFLI | FileFormat::GZIP => Box::new(MultiGzDecoder::new(w)), + _ => unreachable!(), + } +} + +// C++ FFI + +pub fn compress_fd(format: FileFormat, in_fd: RawFd, out_fd: RawFd) { + let mut in_file = unsafe { ManuallyDrop::new(File::from_raw_fd(in_fd)) }; + let mut out_file = unsafe { ManuallyDrop::new(File::from_raw_fd(out_fd)) }; + + let mut encoder = get_encoder(format, out_file.deref_mut()); + let _: LoggedResult<()> = try { + std::io::copy(in_file.deref_mut(), encoder.as_mut())?; + encoder.finish()?; + }; +} + +pub fn decompress_bytes_fd(format: FileFormat, in_bytes: &[u8], in_fd: RawFd, out_fd: RawFd) { + let mut in_file = unsafe { ManuallyDrop::new(File::from_raw_fd(in_fd)) }; + let mut out_file = unsafe { ManuallyDrop::new(File::from_raw_fd(out_fd)) }; + + let mut decoder = get_decoder(format, out_file.deref_mut()); + let _: LoggedResult<()> = try { + decoder.write_all(in_bytes)?; + std::io::copy(in_file.deref_mut(), decoder.as_mut())?; + decoder.finish()?; + }; +} + +pub fn compress_bytes(format: FileFormat, in_bytes: &[u8], out_fd: RawFd) { + let mut out_file = unsafe { ManuallyDrop::new(File::from_raw_fd(out_fd)) }; + + let mut encoder = get_encoder(format, out_file.deref_mut()); + let _: LoggedResult<()> = try { + encoder.write_all(in_bytes)?; + encoder.finish()?; + }; +} + +pub fn decompress_bytes(format: FileFormat, in_bytes: &[u8], out_fd: RawFd) { + let mut out_file = unsafe { ManuallyDrop::new(File::from_raw_fd(out_fd)) }; + + let mut decoder = get_decoder(format, out_file.deref_mut()); + let _: LoggedResult<()> = try { + decoder.write_all(in_bytes)?; + decoder.finish()?; + }; +} diff --git a/native/src/boot/cpio.rs b/native/src/boot/cpio.rs index d26edb917..1d5875ef0 100644 --- a/native/src/boot/cpio.rs +++ b/native/src/boot/cpio.rs @@ -25,7 +25,8 @@ use base::{ }; use crate::check_env; -use crate::ffi::{unxz, xz}; +use crate::compress::{get_decoder, get_encoder}; +use crate::ffi::FileFormat; use crate::patch::{patch_encryption, patch_verity}; #[derive(FromArgs)] @@ -308,7 +309,7 @@ impl Cpio { } pos += file.write( format!("070701{:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x}", - inode, 0o755, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 0 + inode, 0o755, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 0 ).as_bytes() )?; pos += file.write("TRAILER!!!\0".as_bytes())?; @@ -695,12 +696,16 @@ impl CpioEntry { if self.mode & S_IFMT != S_IFREG { return false; } - let mut compressed = Vec::new(); - if !xz(&self.data, &mut compressed) { + let mut encoder = get_encoder(FileFormat::XZ, Vec::new()); + let Ok(data): std::io::Result> = (try { + encoder.write_all(&self.data)?; + encoder.finish()? + }) else { eprintln!("xz compression failed"); return false; - } - self.data = compressed; + }; + + self.data = data; true } @@ -708,12 +713,17 @@ impl CpioEntry { if self.mode & S_IFMT != S_IFREG { return false; } - let mut decompressed = Vec::new(); - if !unxz(&self.data, &mut decompressed) { - eprintln!("xz decompression failed"); + + let mut decoder = get_decoder(FileFormat::XZ, Vec::new()); + let Ok(data): std::io::Result> = (try { + decoder.write_all(&self.data)?; + decoder.finish()? + }) else { + eprintln!("xz compression failed"); return false; - } - self.data = decompressed; + }; + + self.data = data; true } } diff --git a/native/src/boot/format.cpp b/native/src/boot/format.cpp index 35e5bb35b..b6f656f7b 100644 --- a/native/src/boot/format.cpp +++ b/native/src/boot/format.cpp @@ -1,3 +1,4 @@ +#include "boot-rs.hpp" #include "format.hpp" Name2Fmt name2fmt; @@ -6,88 +7,88 @@ Fmt2Ext fmt2ext; #define CHECKED_MATCH(s) (len >= (sizeof(s) - 1) && BUFFER_MATCH(buf, s)) -format_t check_fmt(const void *buf, size_t len) { +FileFormat check_fmt(const void *buf, size_t len) { if (CHECKED_MATCH(CHROMEOS_MAGIC)) { - return CHROMEOS; + return FileFormat::CHROMEOS; } else if (CHECKED_MATCH(BOOT_MAGIC)) { - return AOSP; + return FileFormat::AOSP; } else if (CHECKED_MATCH(VENDOR_BOOT_MAGIC)) { - return AOSP_VENDOR; + return FileFormat::AOSP_VENDOR; } else if (CHECKED_MATCH(GZIP1_MAGIC) || CHECKED_MATCH(GZIP2_MAGIC)) { - return GZIP; + return FileFormat::GZIP; } else if (CHECKED_MATCH(LZOP_MAGIC)) { - return LZOP; + return FileFormat::LZOP; } else if (CHECKED_MATCH(XZ_MAGIC)) { - return XZ; + return FileFormat::XZ; } else if (len >= 13 && memcmp(buf, "\x5d\x00\x00", 3) == 0 && (((char *)buf)[12] == '\xff' || ((char *)buf)[12] == '\x00')) { - return LZMA; + return FileFormat::LZMA; } else if (CHECKED_MATCH(BZIP_MAGIC)) { - return BZIP2; + return FileFormat::BZIP2; } else if (CHECKED_MATCH(LZ41_MAGIC) || CHECKED_MATCH(LZ42_MAGIC)) { - return LZ4; + return FileFormat::LZ4; } else if (CHECKED_MATCH(LZ4_LEG_MAGIC)) { - return LZ4_LEGACY; + return FileFormat::LZ4_LEGACY; } else if (CHECKED_MATCH(MTK_MAGIC)) { - return MTK; + return FileFormat::MTK; } else if (CHECKED_MATCH(DTB_MAGIC)) { - return DTB; + return FileFormat::DTB; } else if (CHECKED_MATCH(DHTB_MAGIC)) { - return DHTB; + return FileFormat::DHTB; } else if (CHECKED_MATCH(TEGRABLOB_MAGIC)) { - return BLOB; + return FileFormat::BLOB; } else if (len >= 0x28 && memcmp(&((char *)buf)[0x24], ZIMAGE_MAGIC, 4) == 0) { - return ZIMAGE; + return FileFormat::ZIMAGE; } else { - return UNKNOWN; + return FileFormat::UNKNOWN; } } -const char *Fmt2Name::operator[](format_t fmt) { +const char *Fmt2Name::operator[](FileFormat fmt) { switch (fmt) { - case GZIP: + case FileFormat::GZIP: return "gzip"; - case ZOPFLI: + case FileFormat::ZOPFLI: return "zopfli"; - case LZOP: + case FileFormat::LZOP: return "lzop"; - case XZ: + case FileFormat::XZ: return "xz"; - case LZMA: + case FileFormat::LZMA: return "lzma"; - case BZIP2: + case FileFormat::BZIP2: return "bzip2"; - case LZ4: + case FileFormat::LZ4: return "lz4"; - case LZ4_LEGACY: + case FileFormat::LZ4_LEGACY: return "lz4_legacy"; - case LZ4_LG: + case FileFormat::LZ4_LG: return "lz4_lg"; - case DTB: + case FileFormat::DTB: return "dtb"; - case ZIMAGE: + case FileFormat::ZIMAGE: return "zimage"; default: return "raw"; } } -const char *Fmt2Ext::operator[](format_t fmt) { +const char *Fmt2Ext::operator[](FileFormat fmt) { switch (fmt) { - case GZIP: - case ZOPFLI: + case FileFormat::GZIP: + case FileFormat::ZOPFLI: return ".gz"; - case LZOP: + case FileFormat::LZOP: return ".lzo"; - case XZ: + case FileFormat::XZ: return ".xz"; - case LZMA: + case FileFormat::LZMA: return ".lzma"; - case BZIP2: + case FileFormat::BZIP2: return ".bz2"; - case LZ4: - case LZ4_LEGACY: - case LZ4_LG: + case FileFormat::LZ4: + case FileFormat::LZ4_LEGACY: + case FileFormat::LZ4_LG: return ".lz4"; default: return ""; @@ -96,15 +97,15 @@ const char *Fmt2Ext::operator[](format_t fmt) { #define CHECK(s, f) else if (name == s) return f; -format_t Name2Fmt::operator[](std::string_view name) { +FileFormat Name2Fmt::operator[](std::string_view name) { if (0) {} - CHECK("gzip", GZIP) - CHECK("zopfli", ZOPFLI) - CHECK("xz", XZ) - CHECK("lzma", LZMA) - CHECK("bzip2", BZIP2) - CHECK("lz4", LZ4) - CHECK("lz4_legacy", LZ4_LEGACY) - CHECK("lz4_lg", LZ4_LG) - else return UNKNOWN; + CHECK("gzip", FileFormat::GZIP) + CHECK("zopfli", FileFormat::ZOPFLI) + CHECK("xz", FileFormat::XZ) + CHECK("lzma", FileFormat::LZMA) + CHECK("bzip2", FileFormat::BZIP2) + CHECK("lz4", FileFormat::LZ4) + CHECK("lz4_legacy", FileFormat::LZ4_LEGACY) + CHECK("lz4_lg", FileFormat::LZ4_LG) + else return FileFormat::UNKNOWN; } diff --git a/native/src/boot/format.hpp b/native/src/boot/format.hpp index 79dac8fec..cd78bdbe7 100644 --- a/native/src/boot/format.hpp +++ b/native/src/boot/format.hpp @@ -2,33 +2,10 @@ #include -typedef enum { - UNKNOWN, -/* Boot formats */ - CHROMEOS, - AOSP, - AOSP_VENDOR, - DHTB, - BLOB, -/* Compression formats */ - GZIP, - ZOPFLI, - XZ, - LZMA, - BZIP2, - LZ4, - LZ4_LEGACY, - LZ4_LG, -/* Unsupported compression */ - LZOP, -/* Misc */ - MTK, - DTB, - ZIMAGE, -} format_t; +enum class FileFormat : ::std::uint8_t; -#define COMPRESSED(fmt) ((fmt) >= GZIP && (fmt) < LZOP) -#define COMPRESSED_ANY(fmt) ((fmt) >= GZIP && (fmt) <= LZOP) +#define COMPRESSED(fmt) ((+fmt) >= +FileFormat::GZIP && (+fmt) < +FileFormat::LZOP) +#define COMPRESSED_ANY(fmt) ((+fmt) >= +FileFormat::GZIP && (+fmt) <= +FileFormat::LZOP) #define BUFFER_MATCH(buf, s) (memcmp(buf, s, sizeof(s) - 1) == 0) #define BUFFER_CONTAIN(buf, sz, s) (memmem(buf, sz, s, sizeof(s) - 1) != nullptr) @@ -66,20 +43,24 @@ typedef enum { class Fmt2Name { public: - const char *operator[](format_t fmt); + const char *operator[](FileFormat fmt); }; class Fmt2Ext { public: - const char *operator[](format_t fmt); + const char *operator[](FileFormat fmt); }; class Name2Fmt { public: - format_t operator[](std::string_view name); + FileFormat operator[](std::string_view name); }; -format_t check_fmt(const void *buf, size_t len); +FileFormat check_fmt(const void *buf, size_t len); + +static inline FileFormat check_fmt(rust::Slice bytes) { + return check_fmt(bytes.data(), bytes.size()); +} extern Name2Fmt name2fmt; extern Fmt2Name fmt2name; diff --git a/native/src/boot/lib.rs b/native/src/boot/lib.rs index e244ccc54..6265dec00 100644 --- a/native/src/boot/lib.rs +++ b/native/src/boot/lib.rs @@ -4,15 +4,15 @@ #![feature(try_blocks)] pub use base; +use compress::{compress_bytes, compress_fd, decompress_bytes, decompress_bytes_fd}; use cpio::cpio_commands; use dtb::dtb_commands; -pub use libbz2_rs_sys::*; -pub use libz_rs_sys::*; use patch::hexpatch; use payload::extract_boot_from_payload; use sign::{SHA, get_sha, sha1_hash, sha256_hash, sign_boot_image, verify_boot_image}; use std::env; +mod compress; mod cpio; mod dtb; mod patch; @@ -24,6 +24,31 @@ mod sign; #[cxx::bridge] pub mod ffi { + enum FileFormat { + UNKNOWN, + /* Boot formats */ + CHROMEOS, + AOSP, + AOSP_VENDOR, + DHTB, + BLOB, + /* Compression formats */ + GZIP, + ZOPFLI, + XZ, + LZMA, + BZIP2, + LZ4, + LZ4_LEGACY, + LZ4_LG, + /* Unsupported compression */ + LZOP, + /* Misc */ + MTK, + DTB, + ZIMAGE, + } + unsafe extern "C++" { include!("../base/include/base.hpp"); @@ -33,10 +58,8 @@ pub mod ffi { } unsafe extern "C++" { - include!("compress.hpp"); - fn decompress(buf: &[u8], fd: i32) -> bool; - fn xz(buf: &[u8], out: &mut Vec) -> bool; - fn unxz(buf: &[u8], out: &mut Vec) -> bool; + include!("format.hpp"); + fn check_fmt(buf: &[u8]) -> FileFormat; include!("bootimg.hpp"); #[cxx_name = "boot_img"] @@ -57,6 +80,10 @@ pub mod ffi { fn sha256_hash(data: &[u8], out: &mut [u8]); fn hexpatch(file: &[u8], from: &[u8], to: &[u8]) -> bool; + fn compress_fd(format: FileFormat, in_fd: i32, out_fd: i32); + fn compress_bytes(format: FileFormat, in_bytes: &[u8], out_fd: i32); + fn decompress_bytes(format: FileFormat, in_bytes: &[u8], out_fd: i32); + fn decompress_bytes_fd(format: FileFormat, in_bytes: &[u8], in_fd: i32, out_fd: i32); } #[namespace = "rust"] diff --git a/native/src/boot/main.cpp b/native/src/boot/main.cpp index 4972f3820..21771cfa8 100644 --- a/native/src/boot/main.cpp +++ b/native/src/boot/main.cpp @@ -2,22 +2,12 @@ #include "boot-rs.hpp" #include "magiskboot.hpp" -#include "compress.hpp" using namespace std; -#ifdef USE_CRT0 -__BEGIN_DECLS -int musl_vfprintf(FILE *stream, const char *format, va_list arg); -int vfprintf(FILE *stream, const char *format, va_list arg) { - return musl_vfprintf(stream, format, arg); -} -__END_DECLS -#endif - static void print_formats() { - for (int fmt = GZIP; fmt < LZOP; ++fmt) { - fprintf(stderr, "%s ", fmt2name[(format_t) fmt]); + for (int fmt = +FileFormat::GZIP; fmt < +FileFormat::LZOP; ++fmt) { + fprintf(stderr, "%s ", fmt2name[(FileFormat) fmt]); } } @@ -126,6 +116,93 @@ Supported actions: exit(1); } +static void decompress(char *infile, const char *outfile) { + bool in_std = infile == "-"sv; + bool rm_in = false; + + int in_fd = in_std ? STDIN_FILENO : xopen(infile, O_RDONLY); + int out_fd = -1; + + uint8_t buf[4096]; + size_t len = read(in_fd, buf, sizeof(buf)); + FileFormat type = check_fmt(buf, len); + + fprintf(stderr, "Detected format: [%s]\n", fmt2name[type]); + + if (!COMPRESSED(type)) + LOGE("Input file is not a supported compressed type!\n"); + + // If user does not provide outfile, infile has to be either + // .[ext], or '-'. Outfile will be either or '-'. + // If the input does not have proper format, abort. + + char *ext = nullptr; + if (outfile == nullptr) { + outfile = infile; + if (!in_std) { + ext = strrchr(infile, '.'); + if (ext == nullptr || strcmp(ext, fmt2ext[type]) != 0) + LOGE("Input file is not a supported type!\n"); + + // Strip out extension and remove input + *ext = '\0'; + rm_in = true; + fprintf(stderr, "Decompressing to [%s]\n", outfile); + } + } + + out_fd = outfile == "-"sv ? + STDOUT_FILENO : + xopen(outfile, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (ext) *ext = '.'; + + decompress_bytes_fd(type, byte_view{ buf, len }, in_fd, out_fd); + + if (in_fd != STDIN_FILENO) close(in_fd); + if (out_fd != STDOUT_FILENO) close(out_fd); + + if (rm_in) + unlink(infile); +} + +static void compress(const char *method, const char *infile, const char *outfile) { + FileFormat fmt = name2fmt[method]; + if (fmt == FileFormat::UNKNOWN) + LOGE("Unknown compression method: [%s]\n", method); + + bool in_std = infile == "-"sv; + bool rm_in = false; + + int in_fd = in_std ? STDIN_FILENO : xopen(infile, O_RDONLY); + int out_fd = -1; + + if (outfile == nullptr) { + if (in_std) { + out_fd = STDOUT_FILENO; + } else { + // If user does not provide outfile and infile is not + // STDIN, output to .[ext] + string tmp(infile); + tmp += fmt2ext[fmt]; + out_fd = xopen(tmp.data(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + fprintf(stderr, "Compressing to [%s]\n", tmp.data()); + rm_in = true; + } + } else { + out_fd = outfile == "-"sv ? + STDOUT_FILENO : + xopen(outfile, O_WRONLY | O_CREAT | O_TRUNC, 0644); + } + + compress_fd(fmt, in_fd, out_fd); + + if (in_fd != STDIN_FILENO) close(in_fd); + if (out_fd != STDOUT_FILENO) close(out_fd); + + if (rm_in) + unlink(infile); +} + int main(int argc, char *argv[]) { cmdline_logging(); umask(0); diff --git a/native/src/boot/payload.rs b/native/src/boot/payload.rs index f6be9eddb..c43d626bc 100644 --- a/native/src/boot/payload.rs +++ b/native/src/boot/payload.rs @@ -1,16 +1,14 @@ +use byteorder::{BigEndian, ReadBytesExt}; +use quick_protobuf::{BytesReader, MessageRead}; use std::{ fs::File, io::{BufReader, Read, Seek, SeekFrom, Write}, - os::fd::{AsRawFd, FromRawFd}, + os::fd::FromRawFd, }; -use byteorder::{BigEndian, ReadBytesExt}; -use quick_protobuf::{BytesReader, MessageRead}; - -use crate::{ - ffi, - proto::update_metadata::{DeltaArchiveManifest, mod_InstallOperation::Type}, -}; +use crate::compress::get_decoder; +use crate::ffi::check_fmt; +use crate::proto::update_metadata::{DeltaArchiveManifest, mod_InstallOperation::Type}; use base::{ LoggedError, LoggedResult, ReadSeekExt, ResultExt, Utf8CStr, WriteExt, error, ffi::Utf8CStrRef, }; @@ -168,9 +166,12 @@ fn do_extract_boot_from_payload( } Type::REPLACE_BZ | Type::REPLACE_XZ => { out_file.seek(SeekFrom::Start(out_offset))?; - if !ffi::decompress(data, out_file.as_raw_fd()) { + let mut decoder = get_decoder(check_fmt(data), &mut out_file); + let Ok(_): std::io::Result<()> = (try { + decoder.write_all(data)?; + }) else { return Err(bad_payload!("decompression failed")); - } + }; } _ => return Err(bad_payload!("unsupported operation type")), }; diff --git a/native/src/external/Android.mk b/native/src/external/Android.mk index 1e9e6baa9..47d0cc5bf 100644 --- a/native/src/external/Android.mk +++ b/native/src/external/Android.mk @@ -218,27 +218,6 @@ LOCAL_SRC_FILES := \ lsplt/lsplt/src/main/jni/lsplt.cc include $(BUILD_STATIC_LIBRARY) -# libzopfli.a -include $(CLEAR_VARS) -LOCAL_MODULE:= libzopfli -LOCAL_C_INCLUDES := $(LOCAL_PATH)/zopfli/src -LOCAL_EXPORT_C_INCLUDES := $(LOCAL_C_INCLUDES) -LOCAL_CFLAGS := -Wall -Werror -Wno-unused -Wno-unused-parameter -LOCAL_SRC_FILES := \ - zopfli/src/zopfli/blocksplitter.c \ - zopfli/src/zopfli/cache.c \ - zopfli/src/zopfli/deflate.c \ - zopfli/src/zopfli/gzip_container.c \ - zopfli/src/zopfli/hash.c \ - zopfli/src/zopfli/katajainen.c \ - zopfli/src/zopfli/lz77.c \ - zopfli/src/zopfli/squeeze.c \ - zopfli/src/zopfli/tree.c \ - zopfli/src/zopfli/util.c \ - zopfli/src/zopfli/zlib_container.c \ - zopfli/src/zopfli/zopfli_lib.c -include $(BUILD_STATIC_LIBRARY) - CWD := $(LOCAL_PATH) include $(CWD)/system_properties/Android.mk include $(CWD)/libcxx/Android.mk diff --git a/native/src/external/crt0 b/native/src/external/crt0 index 0ca1dea7e..f7f5d092b 160000 --- a/native/src/external/crt0 +++ b/native/src/external/crt0 @@ -1 +1 @@ -Subproject commit 0ca1dea7e4e741b48fe94697b563cff712322591 +Subproject commit f7f5d092b28e8ba8ae073f2be72cc6cb96c52843 diff --git a/native/src/external/lz4-sys/Cargo.toml b/native/src/external/lz4-sys/Cargo.toml new file mode 100644 index 000000000..395fa31ed --- /dev/null +++ b/native/src/external/lz4-sys/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "lz4-sys" +license = "MIT" +version = "1.11.1+lz4-1.10.0" +authors = [ "Jens Heyens ", "Artem V. Navrotskiy ", "Patrick Marks "] +description = "Rust LZ4 sys package." +repository = "https://github.com/10xGenomics/lz4-rs" + +[dependencies] +libc = "0.2" diff --git a/native/src/external/lz4-sys/src/lib.rs b/native/src/external/lz4-sys/src/lib.rs new file mode 100644 index 000000000..3edef5eb5 --- /dev/null +++ b/native/src/external/lz4-sys/src/lib.rs @@ -0,0 +1,480 @@ +#![allow(unexpected_cfgs)] +#![no_std] +extern crate libc; + +#[cfg(not(all( + target_arch = "wasm32", + not(any(target_env = "wasi", target_os = "wasi")) +)))] +pub use libc::{c_char, c_int, c_uint, c_ulonglong, c_void, size_t}; + +#[cfg(all( + target_arch = "wasm32", + not(any(target_env = "wasi", target_os = "wasi")) +))] +extern crate alloc; + +#[cfg(all( + target_arch = "wasm32", + not(any(target_env = "wasi", target_os = "wasi")) +))] +mod wasm_shim; + +#[cfg(all( + target_arch = "wasm32", + not(any(target_env = "wasi", target_os = "wasi")) +))] +extern crate std; + +#[cfg(all( + target_arch = "wasm32", + not(any(target_env = "wasi", target_os = "wasi")) +))] +pub use std::os::raw::{c_char, c_int, c_uint, c_ulonglong, c_void}; + +#[cfg(all( + target_arch = "wasm32", + not(any(target_env = "wasi", target_os = "wasi")) +))] +#[allow(non_camel_case_types)] +pub type size_t = usize; + +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct LZ4FCompressionContext(pub *mut c_void); +unsafe impl Send for LZ4FCompressionContext {} + +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct LZ4FDecompressionContext(pub *mut c_void); +unsafe impl Send for LZ4FDecompressionContext {} + +pub type LZ4FErrorCode = size_t; + +#[derive(Clone, Debug)] +#[repr(u32)] +pub enum BlockSize { + Default = 0, // Default - 64KB + Max64KB = 4, + Max256KB = 5, + Max1MB = 6, + Max4MB = 7, +} + +impl BlockSize { + pub fn get_size(&self) -> usize { + match self { + &BlockSize::Default | &BlockSize::Max64KB => 64 * 1024, + &BlockSize::Max256KB => 256 * 1024, + &BlockSize::Max1MB => 1 * 1024 * 1024, + &BlockSize::Max4MB => 4 * 1024 * 1024, + } + } +} + +#[derive(Clone, Debug)] +#[repr(u32)] +pub enum BlockMode { + Linked = 0, + Independent, +} + +#[derive(Clone, Debug)] +#[repr(u32)] +pub enum ContentChecksum { + NoChecksum = 0, + ChecksumEnabled, +} + +#[derive(Clone, Debug)] +#[repr(u32)] +pub enum FrameType { + Frame = 0, + SkippableFrame, +} + +#[derive(Clone, Debug)] +#[repr(u32)] +pub enum BlockChecksum { + NoBlockChecksum = 0, + BlockChecksumEnabled, +} + +#[derive(Debug)] +#[repr(C)] +pub struct LZ4FFrameInfo { + pub block_size_id: BlockSize, + pub block_mode: BlockMode, + pub content_checksum_flag: ContentChecksum, + pub frame_type: FrameType, + pub content_size: c_ulonglong, + pub dict_id: c_uint, + pub block_checksum_flag: BlockChecksum, +} + +#[derive(Debug)] +#[repr(C)] +pub struct LZ4FPreferences { + pub frame_info: LZ4FFrameInfo, + pub compression_level: c_uint, // 0 == default (fast mode); values above 16 count as 16 + pub auto_flush: c_uint, // 1 == always flush : reduce need for tmp buffer + pub favor_dec_speed: c_uint, // 1 == favor decompression speed over ratio, requires level 10+ + pub reserved: [c_uint; 3], +} + +#[derive(Debug)] +#[repr(C)] +pub struct LZ4FCompressOptions { + pub stable_src: c_uint, /* 1 == src content will remain available on future calls + * to LZ4F_compress(); avoid saving src content within tmp + * buffer as future dictionary */ + pub reserved: [c_uint; 3], +} + +#[derive(Debug)] +#[repr(C)] +pub struct LZ4FDecompressOptions { + pub stable_dst: c_uint, /* guarantee that decompressed data will still be there on next + * function calls (avoid storage into tmp buffers) */ + pub reserved: [c_uint; 3], +} + +#[derive(Debug)] +#[repr(C)] +pub struct LZ4StreamEncode(c_void); + +#[derive(Debug)] +#[repr(C)] +pub struct LZ4StreamDecode(c_void); + +pub const LZ4F_VERSION: c_uint = 100; + +extern "C" { + + // int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); + #[allow(non_snake_case)] + pub fn LZ4_compress_default( + source: *const c_char, + dest: *mut c_char, + sourceSize: c_int, + maxDestSize: c_int, + ) -> c_int; + + // int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); + #[allow(non_snake_case)] + pub fn LZ4_compress_fast( + source: *const c_char, + dest: *mut c_char, + sourceSize: c_int, + maxDestSize: c_int, + acceleration: c_int, + ) -> c_int; + + // int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel); + #[allow(non_snake_case)] + pub fn LZ4_compress_HC( + src: *const c_char, + dst: *mut c_char, + srcSize: c_int, + dstCapacity: c_int, + compressionLevel: c_int, + ) -> c_int; + + // int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); + #[allow(non_snake_case)] + pub fn LZ4_decompress_safe( + source: *const c_char, + dest: *mut c_char, + compressedSize: c_int, + maxDecompressedSize: c_int, + ) -> c_int; + + // unsigned LZ4F_isError(LZ4F_errorCode_t code); + pub fn LZ4F_isError(code: size_t) -> c_uint; + + // const char* LZ4F_getErrorName(LZ4F_errorCode_t code); + pub fn LZ4F_getErrorName(code: size_t) -> *const c_char; + + // LZ4F_createCompressionContext() : + // The first thing to do is to create a compressionContext object, which will be used in all + // compression operations. + // This is achieved using LZ4F_createCompressionContext(), which takes as argument a version + // and an LZ4F_preferences_t structure. + // The version provided MUST be LZ4F_VERSION. It is intended to track potential version + // differences between different binaries. + // The function will provide a pointer to a fully allocated LZ4F_compressionContext_t object. + // If the result LZ4F_errorCode_t is not zero, there was an error during context creation. + // Object can release its memory using LZ4F_freeCompressionContext(); + // + // LZ4F_errorCode_t LZ4F_createCompressionContext( + // LZ4F_compressionContext_t* LZ4F_compressionContextPtr, + // unsigned version); + pub fn LZ4F_createCompressionContext( + ctx: &mut LZ4FCompressionContext, + version: c_uint, + ) -> LZ4FErrorCode; + + // LZ4F_errorCode_t LZ4F_freeCompressionContext( + // LZ4F_compressionContext_t LZ4F_compressionContext); + pub fn LZ4F_freeCompressionContext(ctx: LZ4FCompressionContext) -> LZ4FErrorCode; + + // LZ4F_compressBegin() : + // will write the frame header into dstBuffer. + // dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header + // size is 19 bytes. + // The LZ4F_preferences_t structure is optional : you can provide NULL as argument, all + // preferences will then be set to default. + // The result of the function is the number of bytes written into dstBuffer for the header + // or an error code (can be tested using LZ4F_isError()) + // + // size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, + // void* dstBuffer, + // size_t dstMaxSize, + // const LZ4F_preferences_t* preferencesPtr); + pub fn LZ4F_compressBegin( + ctx: LZ4FCompressionContext, + dstBuffer: *mut u8, + dstMaxSize: size_t, + preferencesPtr: *const LZ4FPreferences, + ) -> LZ4FErrorCode; + + // LZ4F_compressBound() : + // Provides the minimum size of Dst buffer given srcSize to handle worst case situations. + // preferencesPtr is optional : you can provide NULL as argument, all preferences will then + // be set to default. + // Note that different preferences will produce in different results. + // + // size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); + pub fn LZ4F_compressBound( + srcSize: size_t, + preferencesPtr: *const LZ4FPreferences, + ) -> LZ4FErrorCode; + + // LZ4F_compressUpdate() + // LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + // The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure + // compression completion even in worst case. + // If this condition is not respected, LZ4F_compress() will fail (result is an errorCode) + // You can get the minimum value of dstMaxSize by using LZ4F_compressBound() + // The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + // The result of the function is the number of bytes written into dstBuffer : it can be zero, + // meaning input data was just buffered. + // The function outputs an error code if it fails (can be tested using LZ4F_isError()) + // + // size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, + // void* dstBuffer, + // size_t dstMaxSize, + // const void* srcBuffer, + // size_t srcSize, + // const LZ4F_compressOptions_t* compressOptionsPtr); + pub fn LZ4F_compressUpdate( + ctx: LZ4FCompressionContext, + dstBuffer: *mut u8, + dstMaxSize: size_t, + srcBuffer: *const u8, + srcSize: size_t, + compressOptionsPtr: *const LZ4FCompressOptions, + ) -> size_t; + + // LZ4F_flush() + // Should you need to create compressed data immediately, without waiting for a block + // to be be filled, you can call LZ4_flush(), which will immediately compress any remaining + // data buffered within compressionContext. + // The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + // The result of the function is the number of bytes written into dstBuffer + // (it can be zero, this means there was no data left within compressionContext) + // The function outputs an error code if it fails (can be tested using LZ4F_isError()) + // + // size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, + // void* dstBuffer, + // size_t dstMaxSize, + // const LZ4F_compressOptions_t* compressOptionsPtr); + pub fn LZ4F_flush( + ctx: LZ4FCompressionContext, + dstBuffer: *mut u8, + dstMaxSize: size_t, + compressOptionsPtr: *const LZ4FCompressOptions, + ) -> LZ4FErrorCode; + + // LZ4F_compressEnd() + // When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). + // It will flush whatever data remained within compressionContext (like LZ4_flush()) + // but also properly finalize the frame, with an endMark and a checksum. + // The result of the function is the number of bytes written into dstBuffer + // (necessarily >= 4 (endMark size)) + // The function outputs an error code if it fails (can be tested using LZ4F_isError()) + // The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + // compressionContext can then be used again, starting with LZ4F_compressBegin(). + // + // size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, + // void* dstBuffer, + // size_t dstMaxSize, + // const LZ4F_compressOptions_t* compressOptionsPtr); + pub fn LZ4F_compressEnd( + ctx: LZ4FCompressionContext, + dstBuffer: *mut u8, + dstMaxSize: size_t, + compressOptionsPtr: *const LZ4FCompressOptions, + ) -> LZ4FErrorCode; + + // LZ4F_createDecompressionContext() : + // The first thing to do is to create a decompressionContext object, which will be used + // in all decompression operations. + // This is achieved using LZ4F_createDecompressionContext(). + // The version provided MUST be LZ4F_VERSION. It is intended to track potential version + // differences between different binaries. + // The function will provide a pointer to a fully allocated and initialized + // LZ4F_decompressionContext_t object. + // If the result LZ4F_errorCode_t is not OK_NoError, there was an error during + // context creation. + // Object can release its memory using LZ4F_freeDecompressionContext(); + // + // LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_decompressionContext_t* ctxPtr, + // unsigned version); + pub fn LZ4F_createDecompressionContext( + ctx: &mut LZ4FDecompressionContext, + version: c_uint, + ) -> LZ4FErrorCode; + + // LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t ctx); + pub fn LZ4F_freeDecompressionContext(ctx: LZ4FDecompressionContext) -> LZ4FErrorCode; + + // LZ4F_getFrameInfo() + // This function decodes frame header information, such as blockSize. + // It is optional : you could start by calling directly LZ4F_decompress() instead. + // The objective is to extract header information without starting decompression, typically + // for allocation purposes. + // LZ4F_getFrameInfo() can also be used *after* starting decompression, on a + // valid LZ4F_decompressionContext_t. + // The number of bytes read from srcBuffer will be provided within *srcSizePtr + // (necessarily <= original value). + // You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) + // The function result is an hint of how many srcSize bytes LZ4F_decompress() expects for + // next call, or an error code which can be tested using LZ4F_isError(). + // + // size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t ctx, + // LZ4F_frameInfo_t* frameInfoPtr, + // const void* srcBuffer, size_t* srcSizePtr); + pub fn LZ4F_getFrameInfo( + ctx: LZ4FDecompressionContext, + frameInfoPtr: &mut LZ4FFrameInfo, + srcBuffer: *const u8, + srcSizePtr: &mut size_t, + ) -> LZ4FErrorCode; + + // LZ4F_decompress() + // Call this function repetitively to regenerate data compressed within srcBuffer. + // The function will attempt to decode *srcSizePtr bytes from srcBuffer, into dstBuffer of + // maximum size *dstSizePtr. + // + // The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr + // (necessarily <= original value). + // + // The number of bytes read from srcBuffer will be provided within *srcSizePtr + // (necessarily <= original value). + // If number of bytes read is < number of bytes provided, then decompression operation + // is not completed. It typically happens when dstBuffer is not large enough to contain + // all decoded data. + // LZ4F_decompress() must be called again, starting from where it stopped + // (srcBuffer + *srcSizePtr) + // The function will check this condition, and refuse to continue if it is not respected. + // + // dstBuffer is supposed to be flushed between each call to the function, since its content + // will be overwritten. + // dst arguments can be changed at will with each consecutive call to the function. + // + // The function result is an hint of how many srcSize bytes LZ4F_decompress() expects for + // next call. + // Schematically, it's the size of the current (or remaining) compressed block + header of + // next block. + // Respecting the hint provides some boost to performance, since it does skip intermediate + // buffers. + // This is just a hint, you can always provide any srcSize you want. + // When a frame is fully decoded, the function result will be 0. (no more data expected) + // If decompression failed, function result is an error code, which can be tested + // using LZ4F_isError(). + // + // size_t LZ4F_decompress(LZ4F_decompressionContext_t ctx, + // void* dstBuffer, size_t* dstSizePtr, + // const void* srcBuffer, size_t* srcSizePtr, + // const LZ4F_decompressOptions_t* optionsPtr); + pub fn LZ4F_decompress( + ctx: LZ4FDecompressionContext, + dstBuffer: *mut u8, + dstSizePtr: &mut size_t, + srcBuffer: *const u8, + srcSizePtr: &mut size_t, + optionsPtr: *const LZ4FDecompressOptions, + ) -> LZ4FErrorCode; + + // int LZ4_versionNumber(void) + pub fn LZ4_versionNumber() -> c_int; + + // int LZ4_compressBound(int isize) + pub fn LZ4_compressBound(size: c_int) -> c_int; + + // LZ4_stream_t* LZ4_createStream(void) + pub fn LZ4_createStream() -> *mut LZ4StreamEncode; + + // int LZ4_compress_continue(LZ4_stream_t* LZ4_streamPtr, + // const char* source, + // char* dest, + // int inputSize) + pub fn LZ4_compress_continue( + LZ4_stream: *mut LZ4StreamEncode, + source: *const u8, + dest: *mut u8, + input_size: c_int, + ) -> c_int; + + // int LZ4_freeStream(LZ4_stream_t* LZ4_streamPtr) + pub fn LZ4_freeStream(LZ4_stream: *mut LZ4StreamEncode) -> c_int; + + // int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, + // const char* dictionary, + // int dictSize) + pub fn LZ4_setStreamDecode( + LZ4_stream: *mut LZ4StreamDecode, + dictionary: *const u8, + dict_size: c_int, + ) -> c_int; + + // LZ4_streamDecode_t* LZ4_createStreamDecode(void) + pub fn LZ4_createStreamDecode() -> *mut LZ4StreamDecode; + + // int LZ4_decompress_safe_continue(LZ4_streamDecode_t* LZ4_streamDecode, + // const char* source, + // char* dest, + // int compressedSize, + // int maxDecompressedSize) + pub fn LZ4_decompress_safe_continue( + LZ4_stream: *mut LZ4StreamDecode, + source: *const u8, + dest: *mut u8, + compressed_size: c_int, + max_decompressed_size: c_int, + ) -> c_int; + + // int LZ4_freeStreamDecode(LZ4_streamDecode_t* LZ4_stream) + pub fn LZ4_freeStreamDecode(LZ4_stream: *mut LZ4StreamDecode) -> c_int; + + // LZ4F_resetDecompressionContext() + // In case of an error, the context is left in "undefined" state. + // In which case, it's necessary to reset it, before re-using it. + // This method can also be used to abruptly stop any unfinished decompression, + // and start a new one using same context resources. + pub fn LZ4F_resetDecompressionContext(ctx: LZ4FDecompressionContext); + +} + +#[test] +fn test_version_number() { + unsafe { + LZ4_versionNumber(); + } +} + +#[test] +fn test_frame_info_size() { + assert_eq!(core::mem::size_of::(), 32); +} diff --git a/native/src/external/lz4-sys/src/wasm_shim.rs b/native/src/external/lz4-sys/src/wasm_shim.rs new file mode 100644 index 000000000..e6bbfb53e --- /dev/null +++ b/native/src/external/lz4-sys/src/wasm_shim.rs @@ -0,0 +1,123 @@ +//! A shim for the libc functions used in lz4-rs that are not available when building for wasm +//! targets. Adapted from the shim present in the [zstd](https://github.com/gyscos/zstd-rs) crate. +//! zstd-rs license here: +//! The MIT License (MIT) +//! Copyright (c) 2016 Alexandre Bury +//! +//! Permission is hereby granted, free of charge, to any person obtaining a copy of this software +//! and associated documentation files (the "Software"), to deal in the Software without +//! restriction, including without limitation the rights to use, copy, modify, merge, publish, +//! distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the +//! Software is furnished to do so, subject to the following conditions: +//! +//! The above copyright notice and this permission notice shall be included in all copies or +//! substantial portions of the Software. +//! +//! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +//! BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +//! NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +//! DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +//! OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +use alloc::alloc::{alloc, alloc_zeroed, dealloc, Layout}; +use core::ffi::{c_int, c_void}; + +const USIZE_ALIGN: usize = core::mem::align_of::(); +const USIZE_SIZE: usize = core::mem::size_of::(); + +#[no_mangle] +pub extern "C" fn rust_lz4_wasm_shim_malloc(size: usize) -> *mut c_void { + wasm_shim_alloc::(size) +} + +#[no_mangle] +pub extern "C" fn rust_lz4_wasm_shim_memcmp( + str1: *const c_void, + str2: *const c_void, + n: usize, +) -> i32 { + // Safety: function contracts requires str1 and str2 at least `n`-long. + unsafe { + let str1: &[u8] = core::slice::from_raw_parts(str1 as *const u8, n); + let str2: &[u8] = core::slice::from_raw_parts(str2 as *const u8, n); + match str1.cmp(str2) { + core::cmp::Ordering::Less => -1, + core::cmp::Ordering::Equal => 0, + core::cmp::Ordering::Greater => 1, + } + } +} + +#[no_mangle] +pub extern "C" fn rust_lz4_wasm_shim_calloc(nmemb: usize, size: usize) -> *mut c_void { + // note: calloc expects the allocation to be zeroed + wasm_shim_alloc::(nmemb * size) +} + +#[inline] +fn wasm_shim_alloc(size: usize) -> *mut c_void { + // in order to recover the size upon free, we store the size below the allocation + // special alignment is never requested via the malloc API, + // so it's not stored, and usize-alignment is used + // memory layout: [size] [allocation] + + let full_alloc_size = size + USIZE_SIZE; + + unsafe { + let layout = Layout::from_size_align_unchecked(full_alloc_size, USIZE_ALIGN); + + let ptr = if ZEROED { + alloc_zeroed(layout) + } else { + alloc(layout) + }; + + // SAFETY: ptr is usize-aligned and we've allocated sufficient memory + ptr.cast::().write(full_alloc_size); + + ptr.add(USIZE_SIZE).cast() + } +} + +#[no_mangle] +pub unsafe extern "C" fn rust_lz4_wasm_shim_free(ptr: *mut c_void) { + // the layout for the allocation needs to be recovered for dealloc + // - the size must be recovered from directly below the allocation + // - the alignment will always by USIZE_ALIGN + + let alloc_ptr = ptr.sub(USIZE_SIZE); + // SAFETY: the allocation routines must uphold having a valid usize below the provided pointer + let full_alloc_size = alloc_ptr.cast::().read(); + + let layout = Layout::from_size_align_unchecked(full_alloc_size, USIZE_ALIGN); + dealloc(alloc_ptr.cast(), layout); +} + +#[no_mangle] +pub unsafe extern "C" fn rust_lz4_wasm_shim_memcpy( + dest: *mut c_void, + src: *const c_void, + n: usize, +) -> *mut c_void { + core::ptr::copy_nonoverlapping(src as *const u8, dest as *mut u8, n); + dest +} + +#[no_mangle] +pub unsafe extern "C" fn rust_lz4_wasm_shim_memmove( + dest: *mut c_void, + src: *const c_void, + n: usize, +) -> *mut c_void { + core::ptr::copy(src as *const u8, dest as *mut u8, n); + dest +} + +#[no_mangle] +pub unsafe extern "C" fn rust_lz4_wasm_shim_memset( + dest: *mut c_void, + c: c_int, + n: usize, +) -> *mut c_void { + core::ptr::write_bytes(dest as *mut u8, c as u8, n); + dest +} diff --git a/native/src/external/lzma-sys/Cargo.toml b/native/src/external/lzma-sys/Cargo.toml new file mode 100644 index 000000000..d52c94279 --- /dev/null +++ b/native/src/external/lzma-sys/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "lzma-sys" +version = "0.1.20" +authors = ["Alex Crichton "] +license = "MIT/Apache-2.0" +repository = "https://github.com/alexcrichton/xz2-rs" +homepage = "https://github.com/alexcrichton/xz2-rs" +documentation = "https://docs.rs/lzma-sys" +description = """ +Raw bindings to liblzma which contains an implementation of LZMA and xz stream +encoding/decoding. + +High level Rust bindings are available in the `xz2` crate. +""" +categories = ["external-ffi-bindings"] +edition = "2018" + +[dependencies] +libc = "0.2.51" + +[features] +static = [] diff --git a/native/src/external/lzma-sys/src/lib.rs b/native/src/external/lzma-sys/src/lib.rs new file mode 100644 index 000000000..38f8307b0 --- /dev/null +++ b/native/src/external/lzma-sys/src/lib.rs @@ -0,0 +1,359 @@ +#![allow(bad_style)] +#![doc(html_root_url = "https://docs.rs/lzma-sys/0.1")] + +use libc::{c_char, c_uchar, c_void, size_t}; +use std::u64; + +#[cfg(target_env = "msvc")] +#[doc(hidden)] +pub type __enum_ty = libc::c_int; +#[cfg(not(target_env = "msvc"))] +#[doc(hidden)] +pub type __enum_ty = libc::c_uint; +pub type lzma_bool = c_uchar; +pub type lzma_ret = __enum_ty; +pub type lzma_action = __enum_ty; +type lzma_reserved_enum = __enum_ty; +pub type lzma_check = __enum_ty; +pub type lzma_vli = u64; +pub type lzma_mode = __enum_ty; +pub type lzma_match_finder = __enum_ty; + +pub const LZMA_OK: lzma_ret = 0; +pub const LZMA_STREAM_END: lzma_ret = 1; +pub const LZMA_NO_CHECK: lzma_ret = 2; +pub const LZMA_UNSUPPORTED_CHECK: lzma_ret = 3; +pub const LZMA_GET_CHECK: lzma_ret = 4; +pub const LZMA_MEM_ERROR: lzma_ret = 5; +pub const LZMA_MEMLIMIT_ERROR: lzma_ret = 6; +pub const LZMA_FORMAT_ERROR: lzma_ret = 7; +pub const LZMA_OPTIONS_ERROR: lzma_ret = 8; +pub const LZMA_DATA_ERROR: lzma_ret = 9; +pub const LZMA_BUF_ERROR: lzma_ret = 10; +pub const LZMA_PROG_ERROR: lzma_ret = 11; + +pub const LZMA_RUN: lzma_action = 0; +pub const LZMA_SYNC_FLUSH: lzma_action = 1; +pub const LZMA_FULL_FLUSH: lzma_action = 2; +pub const LZMA_FULL_BARRIER: lzma_action = 4; +pub const LZMA_FINISH: lzma_action = 3; + +pub const LZMA_CHECK_NONE: lzma_check = 0; +pub const LZMA_CHECK_CRC32: lzma_check = 1; +pub const LZMA_CHECK_CRC64: lzma_check = 4; +pub const LZMA_CHECK_SHA256: lzma_check = 10; + +pub const LZMA_MODE_FAST: lzma_mode = 1; +pub const LZMA_MODE_NORMAL: lzma_mode = 2; + +pub const LZMA_MF_HC3: lzma_match_finder = 0x03; +pub const LZMA_MF_HC4: lzma_match_finder = 0x04; +pub const LZMA_MF_BT2: lzma_match_finder = 0x12; +pub const LZMA_MF_BT3: lzma_match_finder = 0x13; +pub const LZMA_MF_BT4: lzma_match_finder = 0x14; + +pub const LZMA_TELL_NO_CHECK: u32 = 0x01; +pub const LZMA_TELL_UNSUPPORTED_CHECK: u32 = 0x02; +pub const LZMA_TELL_ANY_CHECK: u32 = 0x04; +pub const LZMA_IGNORE_CHECK: u32 = 0x10; +pub const LZMA_CONCATENATED: u32 = 0x08; + +pub const LZMA_PRESET_DEFAULT: u32 = 6; +pub const LZMA_PRESET_LEVEL_MASK: u32 = 0x1f; +pub const LZMA_PRESET_EXTREME: u32 = 1 << 31; + +pub const LZMA_DICT_SIZE_MIN: u32 = 4096; +pub const LZMA_DICT_SIZE_DEFAULT: u32 = 1 << 23; + +pub const LZMA_LCLP_MIN: u32 = 0; +pub const LZMA_LCLP_MAX: u32 = 4; +pub const LZMA_LC_DEFAULT: u32 = 3; + +pub const LZMA_LP_DEFAULT: u32 = 0; + +pub const LZMA_PB_MIN: u32 = 0; +pub const LZMA_PB_MAX: u32 = 4; +pub const LZMA_PB_DEFAULT: u32 = 2; + +pub const LZMA_BACKWARD_SIZE_MIN: lzma_vli = 4; +pub const LZMA_BACKWARD_SIZE_MAX: lzma_vli = 1 << 34; + +pub const LZMA_VLI_MAX: lzma_vli = u64::MAX / 2; +pub const LZMA_VLI_UNKNOWN: lzma_vli = u64::MAX; +pub const LZMA_VLI_BYTES_MAX: usize = 9; + +pub const LZMA_FILTER_X86: lzma_vli = 0x04; +pub const LZMA_FILTER_POWERPC: lzma_vli = 0x05; +pub const LZMA_FILTER_IA64: lzma_vli = 0x06; +pub const LZMA_FILTER_ARM: lzma_vli = 0x07; +pub const LZMA_FILTER_ARMTHUMB: lzma_vli = 0x08; +pub const LZMA_FILTER_SPARC: lzma_vli = 0x09; +pub const LZMA_FILTER_LZMA1: lzma_vli = 0x4000000000000001; +pub const LZMA_FILTER_LZMA2: lzma_vli = 0x21; + +#[repr(C)] +pub struct lzma_allocator { + pub alloc: Option *mut c_void>, + pub free: Option, + pub opaque: *mut c_void, +} + +pub enum lzma_internal {} + +#[repr(C)] +pub struct lzma_stream { + pub next_in: *const u8, + pub avail_in: size_t, + pub total_in: u64, + pub next_out: *mut u8, + pub avail_out: size_t, + pub total_out: u64, + pub allocator: *const lzma_allocator, + + internal: *mut lzma_internal, + reserved_ptr1: *mut c_void, + reserved_ptr2: *mut c_void, + reserved_ptr3: *mut c_void, + reserved_ptr4: *mut c_void, + reserved_int1: u64, + reserved_int2: u64, + reserved_int3: size_t, + reserved_int4: size_t, + reserved_enum1: lzma_reserved_enum, + reserved_enum2: lzma_reserved_enum, +} + +#[repr(C)] +pub struct lzma_filter { + pub id: lzma_vli, + pub options: *mut c_void, +} + +#[repr(C)] +pub struct lzma_mt { + pub flags: u32, + pub threads: u32, + pub block_size: u64, + pub timeout: u32, + pub preset: u32, + pub filters: *const lzma_filter, + pub check: lzma_check, + + reserved_enum1: lzma_reserved_enum, + reserved_enum2: lzma_reserved_enum, + reserved_enum3: lzma_reserved_enum, + reserved_int1: u32, + reserved_int2: u32, + reserved_int3: u32, + reserved_int4: u32, + reserved_int5: u64, + reserved_int6: u64, + reserved_int7: u64, + reserved_int8: u64, + reserved_ptr1: *mut c_void, + reserved_ptr2: *mut c_void, + reserved_ptr3: *mut c_void, + reserved_ptr4: *mut c_void, +} + +#[repr(C)] +#[derive(Copy, Clone)] +pub struct lzma_options_lzma { + pub dict_size: u32, + pub preset_dict: *const u8, + pub preset_dict_size: u32, + pub lc: u32, + pub lp: u32, + pub pb: u32, + pub mode: lzma_mode, + pub nice_len: u32, + pub mf: lzma_match_finder, + pub depth: u32, + + reserved_int1: u32, + reserved_int2: u32, + reserved_int3: u32, + reserved_int4: u32, + reserved_int5: u32, + reserved_int6: u32, + reserved_int7: u32, + reserved_int8: u32, + reserved_enum1: lzma_reserved_enum, + reserved_enum2: lzma_reserved_enum, + reserved_enum3: lzma_reserved_enum, + reserved_enum4: lzma_reserved_enum, + reserved_ptr1: *mut c_void, + reserved_ptr2: *mut c_void, +} + +#[repr(C)] +pub struct lzma_stream_flags { + pub version: u32, + pub backward_size: lzma_vli, + pub check: lzma_check, + + reserved_enum1: lzma_reserved_enum, + reserved_enum2: lzma_reserved_enum, + reserved_enum3: lzma_reserved_enum, + reserved_enum4: lzma_reserved_enum, + reserved_bool1: lzma_bool, + reserved_bool2: lzma_bool, + reserved_bool3: lzma_bool, + reserved_bool4: lzma_bool, + reserved_bool5: lzma_bool, + reserved_bool6: lzma_bool, + reserved_bool7: lzma_bool, + reserved_bool8: lzma_bool, + reserved_int1: u32, + reserved_int2: u32, +} + +#[repr(C)] +pub struct lzma_options_bcj { + pub start_offset: u32, +} + +extern "C" { + pub fn lzma_code(strm: *mut lzma_stream, action: lzma_action) -> lzma_ret; + pub fn lzma_end(strm: *mut lzma_stream); + pub fn lzma_get_progress(strm: *mut lzma_stream, progress_in: *mut u64, progress_out: *mut u64); + pub fn lzma_memusage(strm: *const lzma_stream) -> u64; + pub fn lzma_memlimit_get(strm: *const lzma_stream) -> u64; + pub fn lzma_memlimit_set(strm: *mut lzma_stream, memlimit: u64) -> lzma_ret; + + pub fn lzma_easy_encoder_memusage(preset: u32) -> u64; + pub fn lzma_easy_decoder_memusage(preset: u32) -> u64; + pub fn lzma_easy_encoder(strm: *mut lzma_stream, preset: u32, check: lzma_check) -> lzma_ret; + pub fn lzma_easy_buffer_encode( + preset: u32, + check: lzma_check, + allocator: *const lzma_allocator, + input: *const u8, + in_size: size_t, + out: *mut u8, + out_pos: *mut size_t, + out_size: size_t, + ) -> lzma_ret; + + pub fn lzma_stream_encoder( + strm: *mut lzma_stream, + filters: *const lzma_filter, + check: lzma_check, + ) -> lzma_ret; + pub fn lzma_stream_encoder_mt_memusage(options: *const lzma_mt) -> u64; + pub fn lzma_stream_encoder_mt(strm: *mut lzma_stream, options: *const lzma_mt) -> lzma_ret; + + pub fn lzma_alone_encoder( + strm: *mut lzma_stream, + options: *const lzma_options_lzma, + ) -> lzma_ret; + + pub fn lzma_stream_buffer_bound(uncompressed_size: size_t) -> size_t; + pub fn lzma_stream_buffer_encode( + filters: *mut lzma_filter, + check: lzma_check, + allocator: *const lzma_allocator, + input: *const u8, + in_size: size_t, + out: *mut u8, + out_pos: *mut size_t, + out_size: size_t, + ) -> lzma_ret; + + pub fn lzma_stream_decoder(strm: *mut lzma_stream, memlimit: u64, flags: u32) -> lzma_ret; + pub fn lzma_auto_decoder(strm: *mut lzma_stream, memlimit: u64, flags: u32) -> lzma_ret; + pub fn lzma_alone_decoder(strm: *mut lzma_stream, memlimit: u64) -> lzma_ret; + pub fn lzma_stream_buffer_decode( + memlimit: *mut u64, + flags: u32, + allocator: *const lzma_allocator, + input: *const u8, + in_pos: *mut size_t, + in_size: size_t, + out: *mut u8, + out_pos: *mut size_t, + out_size: size_t, + ) -> lzma_ret; + + pub fn lzma_check_is_supported(check: lzma_check) -> lzma_bool; + pub fn lzma_check_size(check: lzma_check) -> u32; + + pub fn lzma_crc32(buf: *const u8, size: size_t, crc: u32) -> u32; + pub fn lzma_crc64(buf: *const u8, size: size_t, crc: u64) -> u64; + pub fn lzma_get_check(strm: *const lzma_stream) -> lzma_check; + + pub fn lzma_filter_encoder_is_supported(id: lzma_vli) -> lzma_bool; + pub fn lzma_filter_decoder_is_supported(id: lzma_vli) -> lzma_bool; + pub fn lzma_filters_copy( + src: *const lzma_filter, + dest: *mut lzma_filter, + allocator: *const lzma_allocator, + ) -> lzma_ret; + pub fn lzma_raw_encoder_memusage(filters: *const lzma_filter) -> u64; + pub fn lzma_raw_decoder_memusage(filters: *const lzma_filter) -> u64; + pub fn lzma_raw_encoder(strm: *mut lzma_stream, filters: *const lzma_filter) -> lzma_ret; + pub fn lzma_raw_decoder(strm: *mut lzma_stream, filters: *const lzma_filter) -> lzma_ret; + pub fn lzma_filters_update(strm: *mut lzma_stream, filters: *const lzma_filter) -> lzma_ret; + pub fn lzma_raw_buffer_encode( + filters: *const lzma_filter, + allocator: *const lzma_allocator, + input: *const u8, + in_size: size_t, + out: *mut u8, + out_pos: *mut size_t, + out_size: size_t, + ) -> lzma_ret; + pub fn lzma_raw_buffer_decode( + filters: *const lzma_filter, + allocator: *const lzma_allocator, + input: *const u8, + in_pos: *mut size_t, + in_size: size_t, + out: *mut u8, + out_pos: *mut size_t, + out_size: size_t, + ) -> lzma_ret; + pub fn lzma_properties_size(size: *mut u32, filter: *const lzma_filter) -> lzma_ret; + pub fn lzma_properties_encode(filter: *const lzma_filter, props: *mut u8) -> lzma_ret; + pub fn lzma_properties_decode( + filter: *mut lzma_filter, + allocator: *const lzma_allocator, + props: *const u8, + props_size: size_t, + ) -> lzma_ret; + pub fn lzma_physmem() -> u64; + pub fn lzma_cputhreads() -> u32; + + pub fn lzma_stream_header_encode(options: *const lzma_stream_flags, out: *mut u8) -> lzma_ret; + pub fn lzma_stream_footer_encode(options: *const lzma_stream_flags, out: *mut u8) -> lzma_ret; + pub fn lzma_stream_header_decode(options: *mut lzma_stream_flags, input: *const u8) + -> lzma_ret; + pub fn lzma_stream_footer_decode(options: *mut lzma_stream_flags, input: *const u8) + -> lzma_ret; + pub fn lzma_stream_flags_compare( + a: *const lzma_stream_flags, + b: *const lzma_stream_flags, + ) -> lzma_ret; + + pub fn lzma_version_number() -> u32; + pub fn lzma_version_string() -> *const c_char; + + pub fn lzma_vli_encode( + vli: lzma_vli, + vli_pos: *mut size_t, + out: *mut u8, + out_pos: *mut size_t, + out_size: size_t, + ) -> lzma_ret; + pub fn lzma_vli_decode( + vli: *mut lzma_vli, + vli_pos: *mut size_t, + input: *const u8, + in_pos: *mut size_t, + in_size: size_t, + ) -> lzma_ret; + pub fn lzma_vli_size(vli: lzma_vli) -> u32; + + pub fn lzma_lzma_preset(options: *mut lzma_options_lzma, preset: u32) -> lzma_bool; + pub fn lzma_mf_is_supported(mf: lzma_match_finder) -> lzma_bool; +} diff --git a/native/src/external/zopfli b/native/src/external/zopfli deleted file mode 160000 index 831773bc2..000000000 --- a/native/src/external/zopfli +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 831773bc28e318b91a3255fa12c9fcde1606058b