Mirrors/megabasterd
1
0
Fork 0
mirror of https://github.com/tonikelope/megabasterd.git synced 2025-04-29 22:24:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

7.73

Browse Source 
CBC-MAC optimization
This commit is contained in:
tonikelope 2023-01-26 23:05:31 +01:00
parent 1b076aaa75
commit ee5a5a788d
9 changed files with 120 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
pom.xml
View File

@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.tonikelope</groupId>
<artifactId>MegaBasterd</artifactId>
<version>7.72</version>
<version>7.73</version>
<packaging>jar</packaging>
<repositories>
<repository>
@ -20,6 +20,11 @@
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.11.0</version>
</dependency>
<dependency>
<groupId>org.sejda.webp-imageio</groupId>
<artifactId>webp-imageio-sejda</artifactId>

29
src/main/java/com/tonikelope/megabasterd/ChunkUploader.java
View File

@ -25,6 +25,8 @@ import java.nio.channels.Channels;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.crypto.CipherInputStream;
import org.apache.commons.io.input.QueueInputStream;
import org.apache.commons.io.output.QueueOutputStream;
/**
*
@ -117,6 +119,8 @@ public class ChunkUploader implements Runnable, SecureSingleThreadNotifiable {
byte[] buffer = new byte[MainPanel.DEFAULT_BYTE_BUFFER_SIZE];
byte[] buffer_enc = new byte[MainPanel.DEFAULT_BYTE_BUFFER_SIZE];
boolean fatal_error = false;
int conta_error = 0;
@ -141,9 +145,9 @@ public class ChunkUploader implements Runnable, SecureSingleThreadNotifiable {
chunk_id = _upload.nextChunkId();
long chunk_offset = ChunkWriterManager.calculateChunkOffset(chunk_id, Upload.CHUNK_SIZE_MULTI);
long chunk_offset = ChunkWriterManager.calculateChunkOffset(chunk_id, 1);
long chunk_size = ChunkWriterManager.calculateChunkSize(chunk_id, _upload.getFile_size(), chunk_offset, Upload.CHUNK_SIZE_MULTI);
long chunk_size = ChunkWriterManager.calculateChunkSize(chunk_id, _upload.getFile_size(), chunk_offset, 1);
ChunkWriterManager.checkChunkID(chunk_id, _upload.getFile_size(), chunk_offset);
@ -194,12 +198,25 @@ public class ChunkUploader implements Runnable, SecureSingleThreadNotifiable {
f.seek(chunk_offset);
try ( CipherInputStream cis = new CipherInputStream(new BufferedInputStream(Channels.newInputStream(f.getChannel())), genCrypter("AES", "AES/CTR/NoPadding", _upload.getByte_file_key(), forwardMEGALinkKeyIV(_upload.getByte_file_iv(), chunk_offset))); OutputStream out = new ThrottledOutputStream(con.getOutputStream(), _upload.getMain_panel().getStream_supervisor())) {
ByteArrayOutputStream chunk_mac = new ByteArrayOutputStream();
try ( QueueInputStream qis = new QueueInputStream(); QueueOutputStream qos = qis.newQueueOutputStream(); BufferedInputStream bis = new BufferedInputStream(Channels.newInputStream(f.getChannel())); CipherInputStream cis = new CipherInputStream(qis, genCrypter("AES", "AES/CTR/NoPadding", _upload.getByte_file_key(), forwardMEGALinkKeyIV(_upload.getByte_file_iv(), chunk_offset))); OutputStream out = new ThrottledOutputStream(con.getOutputStream(), _upload.getMain_panel().getStream_supervisor())) {
LOG.log(Level.INFO, "{0} Uploading chunk {1} from worker {2} {3}...", new Object[]{Thread.currentThread().getName(), chunk_id, _id, _upload.getFile_name()});
while (!_exit && tot_bytes_up < chunk_size && (reads = cis.read(buffer)) != -1) {
out.write(buffer, 0, reads);
while (!_exit && tot_bytes_up < chunk_size && (reads = bis.read(buffer)) != -1) {
chunk_mac.write(buffer, 0, reads);
for (int i = 0; i < reads; i++) {
qos.write(buffer[i]);
}
for (int i = 0; i < reads; i++) {
buffer_enc[i] = (byte) cis.read();
}
out.write(buffer_enc, 0, reads);
_upload.getPartialProgress().add((long) reads);
@ -215,6 +232,8 @@ public class ChunkUploader implements Runnable, SecureSingleThreadNotifiable {
}
}
_upload.getMac_generator().CHUNK_QUEUE.put(chunk_offset, chunk_mac);
}
if (!_exit) {

2
src/main/java/com/tonikelope/megabasterd/MainPanel.java
View File

@ -67,7 +67,7 @@ import javax.swing.UIManager;
*/
public final class MainPanel {
public static final String VERSION = "7.72";
public static final String VERSION = "7.73";
public static final boolean FORCE_SMART_PROXY = false; //TRUE FOR DEBUGING SMART PROXY
public static final int THROTTLE_SLICE_SIZE = 16 * 1024;
public static final int DEFAULT_BYTE_BUFFER_SIZE = 16 * 1024;

55
src/main/java/com/tonikelope/megabasterd/MiscTools.java
View File

@ -30,6 +30,7 @@ import java.awt.image.BufferedImage;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@ -48,6 +49,7 @@ import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.IntBuffer;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
@ -239,54 +241,29 @@ public class MiscTools {
public static int[] bin2i32a(byte[] bin) {
int l = (int) (4 * Math.ceil((double) bin.length / 4));
byte[] new_bin = Arrays.copyOfRange(bin, 0, l);
IntBuffer intBuf = ByteBuffer.wrap(bin, 0, l).order(ByteOrder.BIG_ENDIAN).asIntBuffer();
bin = new_bin;
int[] array = new int[intBuf.remaining()];
ByteBuffer bin_buffer = ByteBuffer.wrap(bin);
IntBuffer int_buffer = bin_buffer.asIntBuffer();
intBuf.get(array);
if (int_buffer.hasArray()) {
return int_buffer.array();
} else {
ArrayList<Integer> list = new ArrayList<>();
while (int_buffer.hasRemaining()) {
list.add(int_buffer.get());
}
int[] aux = new int[list.size()];
for (int i = 0; i < aux.length; i++) {
aux[i] = list.get(i);
}
return aux;
}
return array;
}
public static byte[] i32a2bin(int[] i32a) {
ByteBuffer bin_buffer = ByteBuffer.allocate(i32a.length * 4);
IntBuffer int_buffer = bin_buffer.asIntBuffer();
int_buffer.put(i32a);
public static byte[] i32a2bin(int[] values) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
if (bin_buffer.hasArray()) {
return bin_buffer.array();
} else {
ArrayList<Byte> list = new ArrayList<>();
DataOutputStream dos = new DataOutputStream(baos);
while (int_buffer.hasRemaining()) {
list.add(bin_buffer.get());
for (int i = 0; i < values.length; ++i) {
try {
dos.writeInt(values[i]);
} catch (IOException ex) {
Logger.getLogger(MiscTools.class.getName()).log(Level.SEVERE, null, ex);
}
byte[] aux = new byte[list.size()];
for (int i = 0; i < aux.length; i++) {
aux[i] = list.get(i);
}
return aux;
}
return baos.toByteArray();
}
public static BigInteger mpi2big(byte[] s) {

5
src/main/java/com/tonikelope/megabasterd/Upload.java
View File

@ -39,7 +39,6 @@ import javax.swing.JComponent;
public class Upload implements Transference, Runnable, SecureSingleThreadNotifiable {
public static final int WORKERS_DEFAULT = 6;
public static final int CHUNK_SIZE_MULTI = 1; //Otra cosa da errores al reanudar una subida (investigar)
public static final boolean DEFAULT_THUMBNAILS = true;
public static final boolean UPLOAD_LOG = false;
private static final Logger LOG = Logger.getLogger(Upload.class.getName());
@ -62,7 +61,7 @@ public class Upload implements Transference, Runnable, SecureSingleThreadNotifia
private long _last_chunk_id_dispatched;
private final ConcurrentLinkedQueue<Long> _partialProgressQueue;
private final ExecutorService _thread_pool;
private int[] _file_meta_mac;
private volatile int[] _file_meta_mac;
private String _fid;
private boolean _notified;
private volatile String _completion_handler;
@ -1305,7 +1304,7 @@ public class Upload implements Transference, Runnable, SecureSingleThreadNotifia
public long calculateLastUploadedChunk(long bytes_read) {
if (bytes_read > 3584 * 1024) {
return 7 + (long) Math.floor((float) (bytes_read - 3584 * 1024) / (1024 * 1024 * Upload.CHUNK_SIZE_MULTI));
return 7 + (long) Math.floor((float) (bytes_read - 3584 * 1024) / (1024 * 1024 * 1));
} else {
long i = 0, tot = 0;

110
src/main/java/com/tonikelope/megabasterd/UploadMACGenerator.java
View File

@ -11,10 +11,10 @@ package com.tonikelope.megabasterd;
import static com.tonikelope.megabasterd.CryptTools.*;
import static com.tonikelope.megabasterd.MiscTools.*;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.util.HashMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.crypto.BadPaddingException;
@ -33,6 +33,7 @@ public class UploadMACGenerator implements Runnable, SecureSingleThreadNotifiabl
private final Object _secure_notify_lock;
private boolean _notified;
private volatile boolean _exit;
public final ConcurrentHashMap<Long, ByteArrayOutputStream> CHUNK_QUEUE = new ConcurrentHashMap<>();
public UploadMACGenerator(Upload upload) {
_secure_notify_lock = new Object();
@ -122,36 +123,40 @@ public class UploadMACGenerator implements Runnable, SecureSingleThreadNotifiabl
Cipher cryptor = genCrypter("AES", "AES/CBC/NoPadding", _upload.getByte_file_key(), i32a2bin(mac_iv));
try ( BufferedInputStream is = new BufferedInputStream(new FileInputStream(_upload.getFile_name()))) {
int[] chunk_mac = new int[4];
byte[] byte_block = new byte[16];
byte[] chunk_bytes;
if (tot > 0) {
is.skip(tot);
}
try {
while (!_exit && !_upload.isStopped() && !_upload.getMain_panel().isExit()) {
int[] chunk_mac = new int[4];
byte[] byte_block = new byte[16];
int reads;
try {
while (!_exit && !_upload.isStopped() && !_upload.getMain_panel().isExit()) {
long chunk_offset = ChunkWriterManager.calculateChunkOffset(chunk_id, 1);
int reads;
long chunk_size = ChunkWriterManager.calculateChunkSize(chunk_id, _upload.getFile_size(), chunk_offset, 1);
long chunk_offset = ChunkWriterManager.calculateChunkOffset(chunk_id, 1);
ChunkWriterManager.checkChunkID(chunk_id, _upload.getFile_size(), chunk_offset);
long chunk_size = ChunkWriterManager.calculateChunkSize(chunk_id, _upload.getFile_size(), chunk_offset, 1);
while (!CHUNK_QUEUE.containsKey(chunk_offset)) {
MiscTools.pausar(1000);
}
ChunkWriterManager.checkChunkID(chunk_id, _upload.getFile_size(), chunk_offset);
try {
try {
chunk_mac[0] = file_iv[0];
chunk_mac[1] = file_iv[1];
chunk_mac[2] = file_iv[0];
chunk_mac[3] = file_iv[1];
chunk_mac[0] = file_iv[0];
chunk_mac[1] = file_iv[1];
chunk_mac[2] = file_iv[0];
chunk_mac[3] = file_iv[1];
long conta_chunk = 0L;
long conta_chunk = 0L;
try ( ByteArrayOutputStream baos = CHUNK_QUEUE.remove(chunk_offset)) {
chunk_bytes = baos.toByteArray();
}
while (conta_chunk < chunk_size && (reads = is.read(byte_block)) != -1) {
try ( ByteArrayInputStream bais = new ByteArrayInputStream(chunk_bytes)) {
while (conta_chunk < chunk_size && (reads = bais.read(byte_block)) != -1) {
if (reads < byte_block.length) {
for (int i = reads; i < byte_block.length; i++) {
@ -172,45 +177,46 @@ public class UploadMACGenerator implements Runnable, SecureSingleThreadNotifiabl
tot += reads;
}
for (int i = 0; i < file_mac.length; i++) {
file_mac[i] ^= chunk_mac[i];
}
file_mac = bin2i32a(cryptor.doFinal(i32a2bin(file_mac)));
} catch (IOException | IllegalBlockSizeException | BadPaddingException ex) {
LOG.log(Level.SEVERE, ex.getMessage());
}
chunk_id++;
int new_cbc_per = (int) ((((double) tot) / _upload.getFile_size()) * 100);
if (new_cbc_per != cbc_per) {
_upload.getView().updateCBC("CBC-MAC " + String.valueOf(new_cbc_per) + "%");
cbc_per = new_cbc_per;
for (int i = 0; i < file_mac.length; i++) {
file_mac[i] ^= chunk_mac[i];
}
file_mac = bin2i32a(cryptor.doFinal(i32a2bin(file_mac)));
} catch (IllegalBlockSizeException | BadPaddingException ex) {
LOG.log(Level.SEVERE, ex.getMessage());
}
mac = (tot == _upload.getFile_size());
chunk_id++;
} catch (ChunkInvalidException e) {
int new_cbc_per = (int) ((((double) tot) / _upload.getFile_size()) * 100);
mac = true;
}
_upload.setTemp_mac_data(String.valueOf(tot) + "#" + String.valueOf(chunk_id) + "#" + Bin2BASE64(i32a2bin(file_mac)));
if (mac) {
int[] meta_mac = {file_mac[0] ^ file_mac[1], file_mac[2] ^ file_mac[3]};
_upload.setFile_meta_mac(meta_mac);
LOG.log(Level.INFO, "{0} MAC GENERATOR {1} finished MAC CALCULATION. Waiting workers to finish uploading (if any)...", new Object[]{Thread.currentThread().getName(), getUpload().getFile_name()});
if (new_cbc_per != cbc_per) {
_upload.getView().updateCBC("CBC-MAC " + String.valueOf(new_cbc_per) + "%");
cbc_per = new_cbc_per;
}
}
mac = (tot == _upload.getFile_size());
} catch (ChunkInvalidException e) {
mac = true;
}
_upload.setTemp_mac_data(String.valueOf(tot) + "#" + String.valueOf(chunk_id) + "#" + Bin2BASE64(i32a2bin(file_mac)));
if (mac) {
int[] meta_mac = {file_mac[0] ^ file_mac[1], file_mac[2] ^ file_mac[3]};
_upload.setFile_meta_mac(meta_mac);
LOG.log(Level.INFO, "{0} MAC GENERATOR {1} finished MAC CALCULATION. Waiting workers to finish uploading (if any)...", new Object[]{Thread.currentThread().getName(), getUpload().getFile_name()});
}
while (!_exit && !_upload.isStopped() && !_upload.getChunkworkers().isEmpty()) {

15
src/main/java/com/tonikelope/megabasterd/UploadView.form
View File

@ -106,11 +106,13 @@
<Component id="slot_status_label" min="-2" max="-2" attributes="0"/>
</Group>
<EmptySpace min="-2" max="-2" attributes="0"/>
<Group type="103" groupAlignment="3" attributes="0">
<Component id="folder_link_button" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="file_link_button" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="file_size_label" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="open_browser_button" alignment="0" max="32767" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Component id="open_browser_button" alignment="0" max="-2" attributes="0"/>
<Group type="103" groupAlignment="3" attributes="0">
<Component id="folder_link_button" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="file_link_button" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="file_size_label" alignment="3" min="-2" max="-2" attributes="0"/>
</Group>
</Group>
<EmptySpace min="-2" max="-2" attributes="0"/>
<Component id="progress_pbar" min="-2" max="-2" attributes="0"/>
@ -387,9 +389,6 @@
<Image iconType="3" name="/images/icons8-export-30.png"/>
</Property>
<Property name="text" type="java.lang.String" value="Open folder in browser"/>
<Property name="cursor" type="java.awt.Cursor" editor="org.netbeans.modules.form.editors2.CursorEditor">
<Color id="Cursor de Mano"/>
</Property>
<Property name="doubleBuffered" type="boolean" value="true"/>
</Properties>
<Events>

12
src/main/java/com/tonikelope/megabasterd/UploadView.java
View File

@ -319,7 +319,6 @@ public class UploadView extends javax.swing.JPanel implements TransferenceView {
open_browser_button.setFont(new java.awt.Font("Dialog", 1, 16)); // NOI18N
open_browser_button.setIcon(new javax.swing.ImageIcon(getClass().getResource("/images/icons8-export-30.png"))); // NOI18N
open_browser_button.setText("Open folder in browser");
open_browser_button.setCursor(new java.awt.Cursor(java.awt.Cursor.HAND_CURSOR));
open_browser_button.setDoubleBuffered(true);
open_browser_button.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
@ -398,11 +397,12 @@ public class UploadView extends javax.swing.JPanel implements TransferenceView {
.addComponent(file_name_label)
.addComponent(slot_status_label))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(folder_link_button)
.addComponent(file_link_button)
.addComponent(file_size_label)
.addComponent(open_browser_button, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(open_browser_button)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(folder_link_button)
.addComponent(file_link_button)
.addComponent(file_size_label)))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(progress_pbar, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)

BIN
src/main/resources/images/mbasterd_screen.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 187 KiB

After

Width:  |  Height:  |  Size: 190 KiB