Added first pass at shrinkler example

This commit is contained in:
alpine9000
2016-03-15 16:25:03 +11:00
parent 4d34f857aa
commit 90f03cb3c6
56 changed files with 4891 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
build
+6
View File
@@ -0,0 +1,6 @@
repo: dffe960c35de060c4e9767c09f66bb1bbc09f06b
node: 22f9859cf2ceb9a230d4271a0d905b948d23d43c
branch: default
latesttag: null
latesttagdistance: 57
changessincelatesttag: 57
+75
View File
@@ -0,0 +1,75 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
Helper classes to access big-endian Amiga words and longwords.
*/
#pragma once
// A big-endian 16-bit integer with implicit conversions to and from unsigned short
class Word {
unsigned short value;
static unsigned short conv(unsigned short val) {
#ifndef AMIGA
return
((val & 0x00ff) << 8) |
((val & 0xff00) >> 8);
#else
return val;
#endif
}
public:
Word(unsigned short val) : value(conv(val)) {}
Word() : value(0) {}
operator unsigned short() const {
return conv(value);
}
bool operator<(const Word& other) const {
return conv(value) < conv(other.value);
}
unsigned short operator+=(unsigned short a) {
unsigned short res = conv(value) + a;
value = conv(res);
return res;
}
};
// A big-endian 32-bit integer with implicit conversions to and from unsigned int
class Longword {
unsigned int value;
static unsigned int conv(unsigned int val) {
#ifndef AMIGA
return
((val & 0x000000ff) << 24) |
((val & 0x0000ff00) << 8) |
((val & 0x00ff0000) >> 8) |
((val & 0xff000000) >> 24);
#else
return val;
#endif
}
public:
Longword(unsigned int value) : value(conv(value)) {}
Longword() : value(0) {}
operator unsigned int() const {
return conv(value);
}
bool operator<(const Longword& other) const {
return conv(value) < conv(other.value);
}
unsigned int operator+=(unsigned int a) {
unsigned int res = conv(value) + a;
value = conv(res);
return res;
}
};
+35
View File
@@ -0,0 +1,35 @@
To build Shrinkler natively for your unix-like system, type
make
You can cross-build for a particular target using
make PLATFORM=<platform>
where <platform> is one of:
amiga: build for Amiga (build on Cygwin, and see below)
windows-32: build for 32-bit Windows (build on Cygwin)
windows-64: build for 64-bit Windows (build on Cygwin)
native: same as default
native-32: native, forced to 32 bits
native-64: native, forced to 64 bits
To build for Amiga, you will first need to download a few things:
Download
http://amiga.sourceforge.net/phps/logger.php?download=GCC-4.5.0-m68k-amigaos-cygwin.7z
and unpack it to the toolchain directory.
Download
http://sourceforge.net/projects/amiga/files/Compilers/related/include-20090222.lha
and unpack it to the toolchain directory.
Create the directory toolchain/ixemul-sdk, download
http://sourceforge.net/projects/amiga/files/ixemul.library/48.2/ixemul-sdk.lha
and unpack it to the directory you just created.
The Amiga version will need ixemul.library version 48 or later to run.
Enjoy!
+115
View File
@@ -0,0 +1,115 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
Abstract interface for entropy coding.
*/
#pragma once
#include "assert.h"
#include <vector>
using std::vector;
class Coder {
bool cacheable;
bool has_cache;
int number_context_offset;
int n_number_contexts;
vector<vector<unsigned short> > cache;
protected:
Coder() : cacheable(false), has_cache(false)
{}
// Mark coder as cacheable
void setCacheable(bool cacheable) {
this->cacheable = cacheable;
}
public:
// Set parameters for number size cache
void setNumberContexts(int number_context_offset, int n_number_contexts, int max_number) {
if (!cacheable) return;
this->number_context_offset = number_context_offset;
this->n_number_contexts = n_number_contexts;
cache.clear();
for (int context_index = 0 ; context_index < n_number_contexts ; context_index++) {
int base_context = number_context_offset + (context_index << 8);
cache.push_back(vector<unsigned short>());
vector<unsigned short>& c = cache.back();
c.resize(4);
c[2] = code(base_context + 2, 0) + code(base_context + 1, 0);
c[3] = code(base_context + 2, 0) + code(base_context + 1, 1);
int prev_base = 2;
for (int data_bits = 2 ; data_bits < 30 ; data_bits++) {
int base = c.size();
int base_sizedif = - code(base_context + data_bits * 2 - 2, 0)
+ code(base_context + data_bits * 2 - 2, 1)
+ code(base_context + data_bits * 2, 0);
for (int msb = 0 ; msb <= 1 ; msb++) {
int sizedif = base_sizedif + code(base_context + data_bits * 2 - 1, msb);
for (int tail = 0 ; tail < 1 << (data_bits - 1) ; tail++) {
int size = c[prev_base + tail] + sizedif;
c.push_back(size);
if (c.size() > max_number) goto next_context;
}
}
prev_base = base;
}
next_context:;
#if 0
for (int i = 2 ; i < c.size() ; i++) {
assert(c[i] == encodeNumber(base_context, i));
}
#endif
}
has_cache = true;
}
// Number of fractional bits in the bit sizes returned by coding functions.
static const int BIT_PRECISION = 6;
// Code the given bit value in the given context.
// Returns the coded size of the bit (in fractional bits).
virtual int code(int context, int bit) = 0;
// Encode a number >= 2 using a variable-length encoding.
// Returns the coded size of the number (in fractional bits).
int encodeNumber(int base_context, int number) {
assert(number >= 2);
if (has_cache) {
int context_index = (base_context - number_context_offset) >> 8;
vector<unsigned short>& cache_for_context = cache[context_index];
if (number < cache_for_context.size()) {
return cache_for_context[number];
}
}
int size = 0;
int context;
int i;
for (i = 0 ; (4 << i) <= number ; i++) {
context = base_context + (i * 2 + 2);
size += code(context, 1);
}
context = base_context + (i * 2 + 2);
size += code(context, 0);
for (; i >= 0 ; i--) {
int bit = ((number >> i) & 1);
context = base_context + (i * 2 + 1);
size += code(context, bit);
}
return size;
}
virtual ~Coder() {}
};
+60
View File
@@ -0,0 +1,60 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
A dummy entropy coder which counts the occurrences of symbols for estimating
the sizes using a SizeMeasuringCoder during the next compression pass.
*/
#pragma once
#include <vector>
using std::vector;
#include "Coder.h"
struct ContextCounts {
int counts[2];
};
class CountingCoder : public Coder {
vector<ContextCounts> context_counts;
friend class SizeMeasuringCoder;
public:
CountingCoder(int n_contexts) {
struct ContextCounts init_counts = { { 0, 0 } };
context_counts.resize(n_contexts, init_counts);
}
CountingCoder(CountingCoder *old_counts, CountingCoder *new_counts) {
for (int i = 0 ; i < old_counts->context_counts.size() ; i++) {
struct ContextCounts old_count = old_counts->context_counts[i];
struct ContextCounts new_count = new_counts->context_counts[i];
struct ContextCounts mixed_count = { {
(old_count.counts[0] * 3 + new_count.counts[0]) / 4,
(old_count.counts[1] * 3 + new_count.counts[1]) / 4
} };
context_counts.push_back(mixed_count);
}
}
virtual int code(int context_index, int bit) {
context_counts[context_index].counts[bit]++;
return 0;
}
void printRange(FILE *out, int first, int num) {
fprintf(out, "[");
for (int i = 0 ; i < num ; i++) {
if (i > 0) {
fprintf(out, " ");
}
fprintf(out, "%d/%d", context_counts[first + i].counts[0], context_counts[first + i].counts[1]);
}
fprintf(out, "]");
}
};
+239
View File
@@ -0,0 +1,239 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
Cuckoo hash map. Used for mapping offsets to edges in the LZ parser.
*/
#pragma once
#include <utility>
#include <algorithm>
#include <new>
using std::pair;
template <typename V> class CuckooHash;
template <typename V>
class CuckooHashIterator {
const CuckooHash<V>* table;
int index;
CuckooHashIterator(const CuckooHash<V>* table, int index) : table(table), index(index)
{}
void find() {
while (table->element_array[index].first == CuckooHash<V>::UNUSED) index++;
}
friend class CuckooHash<V>;
public:
pair<int, V>& operator*() {
find();
return table->element_array[index];
}
pair<int, V>* operator->() {
find();
return &table->element_array[index];
}
CuckooHashIterator<V> operator++(int) {
find();
return CuckooHashIterator<V>(table, index++);
}
bool operator!=(const CuckooHashIterator<V>& other) {
return index != other.index;
}
};
template <typename V>
class CuckooHash {
public:
typedef int key_type;
typedef pair<key_type, V> value_type;
typedef CuckooHashIterator<V> iterator;
private:
friend class CuckooHashIterator<V>;
typedef unsigned hash_type;
static const key_type UNUSED = 0x80000000;
static const hash_type HASH1_MUL = 0xF230D3A1;
static const hash_type HASH2_MUL = 0x8084027F;
static const int INITIAL_SIZE_LOG = 2;
value_type* element_array;
unsigned n_elements:26;
unsigned hash_shift:6;
int array_size() const {
return 1 << (sizeof(hash_type) * 8 - hash_shift);
}
void init_array() {
int size = array_size();
element_array = new value_type[size];
for (int i = 0 ; i < size ; i++) {
element_array[i].first = UNUSED;
element_array[i].second = V();
}
}
value_type* get_array() {
if (element_array == NULL) {
init_array();
}
return element_array;
}
void init() {
n_elements = 0;
hash_shift = sizeof(hash_type) * 8 - INITIAL_SIZE_LOG;
element_array = NULL;
}
void hashes(key_type key, hash_type& hash1, hash_type& hash2) const {
hash_type f = (key << 1) + 1;
hash1 = (f * HASH1_MUL) >> hash_shift;
hash2 = (f * HASH2_MUL) >> hash_shift;
}
void rehash() {
int old_size = array_size();
value_type* old_array = get_array();
n_elements = 0;
hash_shift--;
init_array();
for (int i = 0 ; i < old_size ; i++) {
if (old_array[i].first != UNUSED) {
(*this)[old_array[i].first] = old_array[i].second;
}
}
delete[] old_array;
}
void insert(hash_type hash, int key, V value, int n) {
value_type* array = get_array();
while (array[hash].first != UNUSED) {
if (--n < 0) {
rehash();
(*this)[key] = value;
return;
}
std::swap(key, array[hash].first);
std::swap(value, array[hash].second);
hash_type hash1;
hash_type hash2;
hashes(key, hash1, hash2);
hash ^= hash1 ^ hash2;
}
array[hash].first = key;
array[hash].second = value;
n_elements++;
}
public:
CuckooHash() {
init();
}
CuckooHash(const CuckooHash& source) {
// We only use copy for array initialization, so just create an empty map
init();
}
~CuckooHash() {
delete[] element_array;
}
void clear() {
delete[] element_array;
init();
}
iterator begin() const {
return CuckooHashIterator<V>(this, 0);
}
iterator end() const {
if (element_array == NULL) {
// Empty
return CuckooHashIterator<V>(this, 0);
}
int index = array_size();
assert(element_array != NULL);
value_type* array = element_array;
while (index > 0 && array[index - 1].first == UNUSED) index--;
return CuckooHashIterator<V>(this, index);
}
int size() const {
return n_elements;
}
bool empty() const {
return size() == 0;
}
int count(int key) const {
if (empty()) return 0;
hash_type hash1;
hash_type hash2;
hashes(key, hash1, hash2);
assert(element_array != NULL);
value_type* array = element_array;
if (array[hash1].first == key || array[hash2].first == key) return 1;
return 0;
}
void erase(int key) {
hash_type hash1;
hash_type hash2;
hashes(key, hash1, hash2);
value_type* array = get_array();
hash_type hash;
if (array[hash1].first == key) {
hash = hash1;
} else if (array[hash2].first == key) {
hash = hash2;
} else {
return;
}
array[hash].first = UNUSED;
array[hash].second = V();
n_elements--;
}
V& operator[](int key) {
hash_type hash1;
hash_type hash2;
hashes(key, hash1, hash2);
value_type* array = get_array();
if (array[hash1].first == key) return array[hash1].second;
if (array[hash2].first == key) return array[hash2].second;
if (array[hash1].first == UNUSED) {
array[hash1].first = key;
array[hash1].second = V();
n_elements++;
return array[hash1].second;
}
if (array[hash2].first == UNUSED) {
array[hash2].first = key;
array[hash2].second = V();
n_elements++;
return array[hash2].second;
}
insert(hash1, key, V(), n_elements);
return (*this)[key];
}
};
+129
View File
@@ -0,0 +1,129 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
Operations on raw data files, including loading, crunching and saving.
*/
#pragma once
#include <cstring>
#include <algorithm>
#include <string>
#include <utility>
#include <algorithm>
using std::make_pair;
using std::max;
using std::min;
using std::pair;
using std::string;
#include "AmigaWords.h"
#include "Pack.h"
#include "RangeDecoder.h"
class DataFile {
vector<unsigned char> data;
vector<unsigned> compress(PackParams *params, RefEdgeFactory *edge_factory, bool show_progress) {
vector<unsigned> pack_buffer;
RangeCoder *range_coder = new RangeCoder(LZEncoder::NUM_CONTEXTS + NUM_RELOC_CONTEXTS, pack_buffer);
// Print compression status header
const char *ordinals[] = { "st", "nd", "rd", "th" };
printf("Original");
for (int p = 1 ; p <= params->iterations ; p++) {
printf(" After %d%s pass", p, ordinals[min(p,4)-1]);
}
printf("\n");
// Crunch the data
range_coder->reset();
packData(&data[0], data.size(), 0, params, range_coder, edge_factory, show_progress);
range_coder->finish();
printf("\n\n");
fflush(stdout);
return pack_buffer;
}
void verify(vector<unsigned>& pack_buffer) {
printf("Verifying... ");
fflush(stdout);
RangeDecoder decoder(LZEncoder::NUM_CONTEXTS + NUM_RELOC_CONTEXTS, pack_buffer);
LZDecoder lzd(&decoder);
// Verify data
bool error = false;
LZVerifier verifier(0, &data[0], data.size(), data.size());
decoder.reset();
decoder.setListener(&verifier);
if (!lzd.decode(verifier)) {
error = true;
}
// Check length
if (!error && verifier.size() != data.size()) {
printf("Verify error: data has incorrect length (%d, should have been %d)!\n", verifier.size(), (int) data.size());
error = true;
}
if (error) {
internal_error();
}
printf("OK\n\n");
}
public:
void load(const char *filename) {
FILE *file;
if ((file = fopen(filename, "rb"))) {
fseek(file, 0, SEEK_END);
int length = ftell(file);
fseek(file, 0, SEEK_SET);
data.resize(length);
if (fread(&data[0], 1, data.size(), file) == data.size()) {
fclose(file);
return;
}
}
printf("Error while reading file %s\n\n", filename);
exit(1);
}
void save(const char *filename) {
FILE *file;
if ((file = fopen(filename, "wb"))) {
if (fwrite(&data[0], 1, data.size(), file) == data.size()) {
fclose(file);
return;
}
}
printf("Error while writing file %s\n\n", filename);
exit(1);
}
int size() {
return data.size();
}
DataFile* crunch(PackParams *params, RefEdgeFactory *edge_factory, bool show_progress) {
vector<unsigned> pack_buffer = compress(params, edge_factory, show_progress);
verify(pack_buffer);
DataFile *ef = new DataFile;
ef->data.resize(pack_buffer.size() * 4, 0);
Longword* dest = (Longword*) (void*) &ef->data[0];
for (int i = 0 ; i < pack_buffer.size() ; i++) {
dest[i] = pack_buffer[i];
}
return ef;
}
};
+38
View File
@@ -0,0 +1,38 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
Abstract interface for entropy decoding.
*/
#pragma once
class Decoder {
public:
// Decode a bit in the given context.
// Returns the decoded bit value.
virtual int decode(int context) = 0;
// Decode a number >= 2 using a variable-length encoding.
// Returns the decoded number.
int decodeNumber(int base_context) {
int context;
int i;
for (i = 0 ;; i++) {
context = base_context + (i * 2 + 2);
if (decode(context) == 0) break;
}
int number = 1;
for (; i >= 0 ; i--) {
context = base_context + (i * 2 + 1);
int bit = decode(context);
number = (number << 1) | bit;
}
return number;
}
virtual ~Decoder() {}
};
+35
View File
@@ -0,0 +1,35 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
Binary Amiga code for the decrunch headers.
The .dat files are generated from the .bin files by the Makefile.
*/
#pragma once
unsigned char Header1[] = {
#include "Header1.dat"
};
unsigned char Header1T[] = {
#include "Header1T.dat"
};
unsigned char Header2[] = {
#include "Header2.dat"
};
unsigned char OverlapHeader[] = {
#include "OverlapHeader.dat"
};
unsigned char OverlapHeaderT[] = {
#include "OverlapHeaderT.dat"
};
unsigned char MiniHeader[] = {
#include "MiniHeader.dat"
};
+258
View File
@@ -0,0 +1,258 @@
; Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
; auto wb\Header1\Header1_End\
; auto wb\Header1T\Header1T_End\
; auto wb\Header2\Header2_End\
INIT_ONE_PROB = $8000
ADJUST_SHIFT = 4
SINGLE_BIT_CONTEXTS = 1
NUM_CONTEXTS = 1536
DUMMY_TEXT_LENGTH = 0
; Exec
LIB_VERSION = 20
Forbid = -132
Permit = -138
FreeMem = -210
OldOpenLibrary = -408
CloseLibrary = -414
CacheClearU = -636
; Dos
Write = -48
Output = -60
align 0,4
Header1:
move.l a3,a4
move.l (a4)+,d2
movem.l d0/a0/a4,-(a7)
.hunk: lsl.l #2,d2
move.l a4,a5
move.l d2,a4
move.l (a4)+,d2
bne.b .hunk
move.l $4.w,a6
jmp (a4)
align 0,4
Header1_End:
Header1T:
move.l a3,a4
move.l (a4)+,d2
movem.l d0/a0/a4,-(a7)
.hunk: lsl.l #2,d2
move.l a4,a5
move.l d2,a4
move.l (a4)+,d2
bne.b .hunk
TextLengthInstr:
move.l #DUMMY_TEXT_LENGTH,d3
move.l $4.w,a6
lea.l DosName(pc),a1
jsr OldOpenLibrary(a6)
move.l d0,a6
jsr Output(a6)
move.l d0,d1
beq.b .noout
lea.l Header1T_End(pc),a0
move.l a0,d2
jsr Write(a6)
.noout: move.l a6,a1
move.l $4.w,a6
jsr CloseLibrary(a6)
jmp (a4)
DosName:
dc.b "dos.library",0
align 0,4
Header1T_End:
Header2:
; Detach last hunk
clr.l -(a5)
; Packed data Start
ContextOffsetInstr:
lea.l Header2_End(pc),a4
; Init range decoder state
moveq.l #1,d1
ror.l #1,d1
moveq.l #1,d3
; Lowest bit of D2 = 0
move.l a3,a2
HunkLoop:
move.l a2,a1
addq.l #4,a1
; A1 = Hunk Data Destination
moveq.l #NUM_CONTEXTS>>4,d6
lsl.l #4,d6
.init: move.w #INIT_ONE_PROB,-(a7)
subq.w #1,d6
bne.b .init
; moveq.l #0,d6
.lit:
addq.b #1,d6
.getlit:
bsr.b GetBit
addx.b d6,d6
bcc.b .getlit
move.b d6,(a1)+
.switch:
bsr.b GetKind
bcc.b .lit
.ref:
moveq.l #-1,d6
bsr.b GetBit
bcs.b .sameoffset
.newref:
moveq.l #3,d6
bsr.b GetNumber
moveq.l #2,d5
sub.l d7,d5
beq.b .hunkend
.sameoffset:
moveq.l #4,d6
bsr.b GetNumber
.copyloop:
move.b (a1,d5.l),(a1)+
subq.l #1,d7
bne.b .copyloop
.afterref:
bsr.b GetKind
bcc.b .lit
bra.b .newref
.hunkend:
; Relocs
move.l a3,d5
RelocHunk:
addq.l #4,d5
move.l a2,a1
.relocloop:
moveq.l #5,d6
bsr.b GetNumber
add.l d7,a1
lsr.l #2,d7
beq.b NextRelocHunk
add.l d5,(a1)
bra.b .relocloop
NextRelocHunk:
move.l d5,a1
move.l -(a1),d5
lsl.l #2,d5
bne.b RelocHunk
NextHunk:
lea.l NUM_CONTEXTS*2(a7),a7
move.l (a2),d4
lsl.l #2,d4
move.l d4,a2
bne.b HunkLoop
End:
cmp.w #37,LIB_VERSION(a6)
blt.b .not204
jsr CacheClearU(a6)
.not204:
jsr Forbid(a6)
lea.l Header2-8(pc),a1
move.l (a1),d0
jsr FreeMem(a6)
movem.l (a7)+,d0/a0
jmp Permit(a6)
GetKind:
move.l a1,d4
moveq.l #1,d6
and.l d4,d6
lsl.w #8,d6
GetBit: bra.b GetBitInner
GetNumber:
; D6 = Number context
; Out: Number in D7
lsl.w #8,d6
.numberloop:
addq.b #2,d6
bsr.b GetBitInner
bcs.b .numberloop
moveq.l #1,d7
subq.b #1,d6
.bitsloop:
bsr.b GetBitInner
addx.l d7,d7
subq.b #2,d6
bcc.b .bitsloop
rts
; D6 = Bit context
; D1 = Input bit buffer
; D2 = Range value
; D3 = Interval size
; Out: Bit in C and X
readbit:
add.l d1,d1
bne.b nonewword
move.l (a4)+,d1
addx.l d1,d1
nonewword:
addx.w d2,d2
add.w d3,d3
GetBitInner:
tst.w d3
bpl.b readbit
lea.l 4+SINGLE_BIT_CONTEXTS*2(a7,d6.l),a5
add.l d6,a5
move.w (a5),d4
; D4 = One prob
lsr.w #ADJUST_SHIFT,d4
sub.w d4,(a5)
add.w (a5),d4
mulu.w d3,d4
swap.w d4
sub.w d4,d2
blo.b .one
.zero:
; oneprob = oneprob * (1 - adjust) = oneprob - oneprob * adjust
sub.w d4,d3
; 0 in C and X
rts
.one:
; onebrob = 1 - (1 - oneprob) * (1 - adjust) = oneprob - oneprob * adjust + adjust
add.w #$ffff>>ADJUST_SHIFT,(a5)
move.w d4,d3
add.w d4,d2
; 1 in C and X
rts
align 0,4
Header2_End:
printv Header1_End-Header1
printv Header1T_End-Header1T
printv Header2_End-Header2
printt
printv TextLengthInstr+2-Header1T
printv ContextOffsetInstr+2-Header2
Binary file not shown.
+1
View File
@@ -0,0 +1 @@
0x28, 0x4B, 0x24, 0x1C, 0x48, 0xE7, 0x80, 0x88, 0xE5, 0x8A, 0x2A, 0x4C, 0x28, 0x42, 0x24, 0x1C, 0x66, 0xF6, 0x2C, 0x78, 0x00, 0x04, 0x4E, 0xD4,
Binary file not shown.
+1
View File
@@ -0,0 +1 @@
0x28, 0x4B, 0x24, 0x1C, 0x48, 0xE7, 0x80, 0x88, 0xE5, 0x8A, 0x2A, 0x4C, 0x28, 0x42, 0x24, 0x1C, 0x66, 0xF6, 0x26, 0x3C, 0x00, 0x00, 0x00, 0x00, 0x2C, 0x78, 0x00, 0x04, 0x43, 0xFA, 0x00, 0x26, 0x4E, 0xAE, 0xFE, 0x68, 0x2C, 0x40, 0x4E, 0xAE, 0xFF, 0xC4, 0x22, 0x00, 0x67, 0x0A, 0x41, 0xFA, 0x00, 0x20, 0x24, 0x08, 0x4E, 0xAE, 0xFF, 0xD0, 0x22, 0x4E, 0x2C, 0x78, 0x00, 0x04, 0x4E, 0xAE, 0xFE, 0x62, 0x4E, 0xD4, 0x64, 0x6F, 0x73, 0x2E, 0x6C, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x00,
Binary file not shown.
+1
View File
@@ -0,0 +1 @@
0x42, 0xA5, 0x49, 0xFA, 0x00, 0xE8, 0x72, 0x01, 0xE2, 0x99, 0x76, 0x01, 0x24, 0x4B, 0x22, 0x4A, 0x58, 0x89, 0x7C, 0x60, 0xE9, 0x8E, 0x3F, 0x3C, 0x80, 0x00, 0x53, 0x46, 0x66, 0xF8, 0x52, 0x06, 0x61, 0x7E, 0xDD, 0x06, 0x64, 0xFA, 0x12, 0xC6, 0x61, 0x6E, 0x64, 0xF2, 0x7C, 0xFF, 0x61, 0x70, 0x65, 0x0A, 0x7C, 0x03, 0x61, 0x6C, 0x7A, 0x02, 0x9A, 0x87, 0x67, 0x12, 0x7C, 0x04, 0x61, 0x62, 0x12, 0xF1, 0x58, 0x00, 0x53, 0x87, 0x66, 0xF8, 0x61, 0x4E, 0x64, 0xD2, 0x60, 0xE4, 0x2A, 0x0B, 0x58, 0x85, 0x22, 0x4A, 0x7C, 0x05, 0x61, 0x4A, 0xD3, 0xC7, 0xE4, 0x8F, 0x67, 0x04, 0xDB, 0x91, 0x60, 0xF2, 0x22, 0x45, 0x2A, 0x21, 0xE5, 0x8D, 0x66, 0xE6, 0x4F, 0xEF, 0x0C, 0x00, 0x28, 0x12, 0xE5, 0x8C, 0x24, 0x44, 0x66, 0x98, 0x0C, 0x6E, 0x00, 0x25, 0x00, 0x14, 0x6D, 0x04, 0x4E, 0xAE, 0xFD, 0x84, 0x4E, 0xAE, 0xFF, 0x7C, 0x43, 0xFA, 0xFF, 0x70, 0x20, 0x11, 0x4E, 0xAE, 0xFF, 0x2E, 0x4C, 0xDF, 0x01, 0x01, 0x4E, 0xEE, 0xFF, 0x76, 0x28, 0x09, 0x7C, 0x01, 0xCC, 0x84, 0xE1, 0x4E, 0x60, 0x22, 0xE1, 0x4E, 0x54, 0x06, 0x61, 0x1C, 0x65, 0xFA, 0x7E, 0x01, 0x53, 0x06, 0x61, 0x14, 0xDF, 0x87, 0x55, 0x06, 0x64, 0xF8, 0x4E, 0x75, 0xD2, 0x81, 0x66, 0x04, 0x22, 0x1C, 0xD3, 0x81, 0xD5, 0x42, 0xD6, 0x43, 0x4A, 0x43, 0x6A, 0xF0, 0x4B, 0xF7, 0x68, 0x06, 0xDB, 0xC6, 0x38, 0x15, 0xE8, 0x4C, 0x99, 0x55, 0xD8, 0x55, 0xC8, 0xC3, 0x48, 0x44, 0x94, 0x44, 0x65, 0x04, 0x96, 0x44, 0x4E, 0x75, 0x06, 0x55, 0x0F, 0xFF, 0x36, 0x04, 0xD4, 0x44, 0x4E, 0x75,
+94
View File
@@ -0,0 +1,94 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
Heap-based priority queue with removal support.
The element type must have an accessible _heap_index integer field.
*/
#pragma once
#include <vector>
#include <functional>
using std::vector;
using std::less;
template <class T>
class Heap {
vector<T> elements;
less<T> compare;
void swap(int i1, int i2) {
T t1 = elements[i1];
T t2 = elements[i2];
elements[i1] = t2;
elements[i2] = t1;
t2->_heap_index = i1;
t1->_heap_index = i2;
}
void up(int i) {
while (i > 0) {
int pi = (i-1)/2;
if (!compare(elements[pi], elements[i])) return;
swap(i, pi);
i = pi;
}
}
void down(int i) {
while (i*2+1 < elements.size()) {
int ci1 = i*2+1;
int ci2 = i*2+2;
int ci = ci2 < elements.size() && compare(elements[ci1], elements[ci2]) ? ci2 : ci1;
if (!compare(elements[i], elements[ci])) return;
swap(i, ci);
i = ci;
}
}
T remove_index(int i) {
T removed = elements[i];
T last = elements[elements.size()-1];
elements[i] = last;
elements.pop_back();
last->_heap_index = i;
down(i);
return removed;
}
public:
Heap() {}
void insert(T t) {
elements.push_back(t);
t->_heap_index = elements.size()-1;
up(elements.size()-1);
}
void remove(T t) {
if (contains(t)) {
remove_index(t->_heap_index);
}
}
T remove_largest() {
return remove_index(0);
}
bool contains(T t) {
return t->_heap_index < elements.size() && elements[t->_heap_index] == t;
}
int size() {
return elements.size();
}
void clear() {
elements.clear();
}
};
+895
View File
@@ -0,0 +1,895 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
Operations on Amiga executables, including loading, parsing,
hunk merging, crunching and saving.
*/
#pragma once
#include <cstring>
#include <algorithm>
#include <string>
#include <utility>
#include <algorithm>
using std::make_pair;
using std::max;
using std::min;
using std::pair;
using std::string;
#include "doshunks.h"
#include "AmigaWords.h"
#include "DecrunchHeaders.h"
#include "Pack.h"
#include "RangeDecoder.h"
#include "LZDecoder.h"
const char *hunktype[HUNK_ABSRELOC16-HUNK_UNIT+1] = {
"UNIT","NAME","CODE","DATA","BSS ","RELOC32","RELOC16","RELOC8",
"EXT","SYMBOL","DEBUG","END","HEADER","","OVERLAY","BREAK",
"DREL32","DREL16","DREL8","LIB","INDEX",
"RELOC32SHORT","RELRELOC32","ABSRELOC16"
};
#define HUNKF_MASK (HUNKF_FAST | HUNKF_CHIP)
#define NUM_RELOC_CONTEXTS 256
class HunkInfo {
public:
HunkInfo() { type = 0; relocentries = 0; }
unsigned type; // HUNK_<type>
unsigned flags; // HUNKF_<flag>
int memsize,datasize; // longwords
int datastart; // longword index in file
int relocstart; // longword index in file
int relocentries; // no. of entries
};
// Compare waste space
class HunkMergeCompare {
vector<HunkInfo>& hunks;
int waste(int h) {
if (hunks[h].type == HUNK_BSS) {
return hunks[h].memsize;
} else {
return hunks[h].memsize - hunks[h].datasize;
}
}
public:
HunkMergeCompare(vector<HunkInfo>& hunks) : hunks(hunks) {}
bool operator()(int h1, int h2) {
return waste(h1) < waste(h2);
}
};
class LZVerifier : public LZReceiver, public CompressedDataReadListener {
int hunk;
unsigned char *data;
int data_length;
int hunk_mem;
int pos;
unsigned char getData(int i) {
if (data == NULL || i >= data_length) return 0;
return data[i];
}
public:
int compressed_longword_count;
int front_overlap_margin;
LZVerifier(int hunk, unsigned char *data, int data_length, int hunk_mem) : hunk(hunk), data(data), data_length(data_length), hunk_mem(hunk_mem), pos(0) {
compressed_longword_count = 0;
front_overlap_margin = 0;
}
bool receiveLiteral(unsigned char lit) {
if (pos >= hunk_mem) {
printf("Verify error: literal at position %d in hunk %d overflows hunk!\n",
pos, hunk);
return false;
}
if (lit != getData(pos)) {
printf("Verify error: literal at position %d in hunk %d has incorrect value (0x%02X, should be 0x%02X)!\n",
pos, hunk, lit, getData(pos));
return false;
}
pos += 1;
return true;
}
bool receiveReference(int offset, int length) {
if (offset < 1 || offset > pos) {
printf("Verify error: reference at position %d in hunk %d has invalid offset (%d)!\n",
pos, hunk, offset);
return false;
}
if (length > hunk_mem - pos) {
printf("Verify error: reference at position %d in hunk %d overflows hunk (length %d, %d bytes past end)!\n",
pos, hunk, length, pos + length - hunk_mem);
return false;
}
for (int i = 0 ; i < length ; i++) {
if (getData(pos - offset + i) != getData(pos + i)) {
printf("Verify error: reference at position %d in hunk %d has incorrect value for byte %d of %d (0x%02X, should be 0x%02X)!\n",
pos, hunk, i, length, getData(pos - offset + i), getData(pos + i));
return false;
}
}
pos += length;
return true;
}
int size() {
return pos;
}
void read(int index) {
// Another longword of compresed data read
int margin = pos - compressed_longword_count * 4;
if (margin > front_overlap_margin) {
front_overlap_margin = margin;
}
compressed_longword_count += 1;
}
};
class HunkFile {
vector<Longword> data;
vector<HunkInfo> hunks;
vector<unsigned> compress_hunks(PackParams *params, bool overlap, bool mini, RefEdgeFactory *edge_factory, bool show_progress) {
int numhunks = hunks.size();
vector<unsigned> pack_buffer;
RangeCoder *range_coder = new RangeCoder(LZEncoder::NUM_CONTEXTS + NUM_RELOC_CONTEXTS, pack_buffer);
// Print compression status header
const char *ordinals[] = { "st", "nd", "rd", "th" };
printf("Hunk Original");
for (int p = 1 ; p <= params->iterations ; p++) {
printf(" After %d%s pass", p, ordinals[min(p,4)-1]);
}
if (!mini) {
printf(" Relocs");
}
printf("\n");
// Crunch the hunks, one by one.
for (int h = 0 ; h < (mini ? 1 : numhunks) ; h++) {
printf("%4d ", h);
range_coder->reset();
switch (hunks[h].type) {
case HUNK_CODE:
case HUNK_DATA:
{
// Pack data
unsigned char *hunk_data = (unsigned char *) &data[hunks[h].datastart];
int hunk_data_length = hunks[h].datasize * 4;
// Trim trailing zeros
while (hunk_data_length > 0 && hunk_data[hunk_data_length - 1] == 0) {
hunk_data_length--;
}
int zero_padding = mini ? 0 : hunks[h].memsize * 4 - hunk_data_length;
packData(hunk_data, hunk_data_length, zero_padding, params, range_coder, edge_factory, show_progress);
}
break;
default:
int zero_padding = mini ? 0 : hunks[h].memsize * 4;
packData(NULL, 0, zero_padding, params, range_coder, edge_factory, show_progress);
break;
}
if (!mini) {
// Reloc table
int reloc_size = 0;
for (int rh = 0 ; rh < numhunks ; rh++) {
vector<int> offsets;
if (hunks[h].relocentries > 0) {
int spos = hunks[h].relocstart;
while (data[spos] != 0) {
int rn = data[spos++];
if (data[spos++] == rh) {
while (rn--) {
offsets.push_back(data[spos++]);
}
} else {
spos += rn;
}
}
sort(offsets.begin(), offsets.end());
}
int last_offset = -4;
for (int ri = 0 ; ri < offsets.size() ; ri++) {
int offset = offsets[ri];
int delta = offset - last_offset;
if (delta < 4) {
printf("\n\nError in input file: overlapping reloc entries.\n\n");
exit(1);
}
reloc_size += range_coder->encodeNumber(LZEncoder::NUM_CONTEXTS, delta);
last_offset = offset;
}
reloc_size += range_coder->encodeNumber(LZEncoder::NUM_CONTEXTS, 2);
}
printf(" %10.3f", reloc_size / (double) (8 << Coder::BIT_PRECISION));
}
printf("\n");
fflush(stdout);
}
range_coder->finish();
printf("\n");
return pack_buffer;
}
vector<pair<int,int> > verify(vector<unsigned>& pack_buffer, bool overlap, bool mini) {
int numhunks = hunks.size();
vector<pair<int,int> > count_and_hunksize;
printf("Verifying... ");
fflush(stdout);
RangeDecoder decoder(LZEncoder::NUM_CONTEXTS + NUM_RELOC_CONTEXTS, pack_buffer);
LZDecoder lzd(&decoder);
for (int h = 0 ; h < (mini ? 1 : numhunks) ; h++) {
unsigned char *hunk_data;
int hunk_data_length = hunks[h].datasize * 4;
if (hunks[h].type != HUNK_BSS) {
// Find hunk data
hunk_data = (unsigned char *) &data[hunks[h].datastart];
if (mini) {
// Trim trailing zeros
while (hunk_data_length > 0 && hunk_data[hunk_data_length - 1] == 0) {
hunk_data_length--;
}
}
} else {
// Signal empty hunk by NULL data pointer
hunk_data = NULL;
}
// Verify data
bool error = false;
LZVerifier verifier(h, hunk_data, hunk_data_length, hunks[h].memsize * sizeof(Longword));
decoder.reset();
decoder.setListener(&verifier);
if (!lzd.decode(verifier)) {
error = true;
}
// Check length
if (!error && !mini && verifier.size() != hunks[h].memsize * sizeof(Longword)) {
printf("Verify error: hunk %d has incorrect length (%d, should have been %d)!\n", h, verifier.size(), hunk_data_length);
error = true;
}
if (error) {
internal_error();
}
if (!mini) {
// Skip relocs
for (int rh = 0 ; rh < numhunks ; rh++) {
int delta;
do {
delta = decoder.decodeNumber(LZEncoder::NUM_CONTEXTS);
} while (delta != 2);
}
}
int margin = verifier.front_overlap_margin;
int count = verifier.compressed_longword_count;
int min_hunksize = (margin == 0 ? 1 : (margin + 3) / 4) + count;
count_and_hunksize.push_back(make_pair(count, min_hunksize));
}
printf("OK\n\n");
return count_and_hunksize;
}
public:
void load(const char *filename) {
FILE *file;
if ((file = fopen(filename, "rb"))) {
fseek(file, 0, SEEK_END);
int length = ftell(file);
fseek(file, 0, SEEK_SET);
if (length & 3) {
printf("File %s has an illegal size!\n\n", filename);
fclose(file);
exit(1);
}
data.resize(length / 4);
if (fread(&data[0], 4, data.size(), file) == data.size()) {
fclose(file);
return;
}
}
printf("Error while reading file %s\n\n", filename);
exit(1);
}
void save(const char *filename) {
FILE *file;
if ((file = fopen(filename, "wb"))) {
if (fwrite(&data[0], 4, data.size(), file) == data.size()) {
fclose(file);
return;
}
}
printf("Error while writing file %s\n\n", filename);
exit(1);
}
int size() {
return data.size() * 4;
}
bool analyze() {
int index = 0;
int length = data.size();
if (data[index++] != HUNK_HEADER) {
printf("No hunk header!\n");
return false;
}
while (data[index++]) {
index += data[index];
if (index >= length) {
printf("Bad hunk header!\n");
return false;
}
}
int numhunks = data[index++];
if (numhunks == 0) {
printf("No hunks!\n");
return false;
}
if (data[index++] != 0 || data[index++] != numhunks-1) {
printf("Unsupported hunk load limits!\n");
return false;
}
hunks.resize(numhunks);
for (int h = 0 ; h < numhunks ; h++) {
hunks[h].memsize = data[index] & 0x0fffffff;
switch (hunks[h].flags = data[index] & 0xf0000000) {
case 0:
case HUNKF_CHIP:
case HUNKF_FAST:
break;
default:
printf("Illegal hunk flags!\n");
return false;
}
index++;
}
// Parse hunks
printf("Hunk Mem Type Mem size Data size Data sum Relocs\n");
for (int h = 0, nh = 0 ; h < numhunks ;) {
unsigned flags = hunks[h].flags, type;
int hunk_length, symlen, n_symbols;
int lh = h;
printf("%4d %s ", h, flags == HUNKF_CHIP ? "CHIP" : flags == HUNKF_FAST ? "FAST" : "ANY ");
int missing_relocs = 0;
const char *note = "";
while (lh == h) {
if (index >= length) {
printf("\nUnexpected end of file!\n");
return false;
}
type = data[index++] & 0x0fffffff;
if (index >= length && type != HUNK_END) {
printf("\nUnexpected end of file!\n");
return false;
}
if (missing_relocs && type != HUNK_RELOC32) {
printf(" %s\n", note);
note = "";
missing_relocs = 0;
}
switch (type) {
case HUNK_UNIT:
case HUNK_NAME:
case HUNK_DEBUG:
printf(" %s (skipped)\n",hunktype[type-HUNK_UNIT]);
hunk_length = data[index++];
index += hunk_length;
break;
case HUNK_SYMBOL:
n_symbols = 0;
symlen = data[index++];
while (symlen > 0) {
n_symbols++;
index += symlen+1;
symlen = data[index++];
}
printf(" SYMBOL (%d entries)\n", n_symbols);
break;
case HUNK_CODE:
case HUNK_DATA:
case HUNK_BSS:
if (nh > h) {
h = nh;
index--;
break;
}
hunks[h].type = type;
hunks[h].datasize = data[index++];
printf("%4s%10d %10d", hunktype[type-HUNK_UNIT], hunks[h].memsize*4, hunks[h].datasize*4);
if (type != HUNK_BSS) {
hunks[h].datastart = index;
index += hunks[h].datasize;
if (hunks[h].datasize > 0) {
int sum = 0;
for (int pos = hunks[h].datastart ; pos < hunks[h].datastart+hunks[h].datasize ; pos++) {
sum += data[pos];
}
printf(" %08x", sum);
} else {
printf(" ");
}
}
if (hunks[h].datasize > hunks[h].memsize) {
note = " Hunk size overflow corrected!";
hunks[h].memsize = hunks[h].datasize;
}
nh = h+1;
missing_relocs = 1;
break;
case HUNK_RELOC32:
hunks[h].relocstart = index;
{
int n,tot = 0;
while ((n = data[index++]) != 0) {
if (n < 0 || index+n+2 >= length || data[index++] >= numhunks) {
printf("\nError in reloc table!\n");
return false;
}
tot += n;
while (n--) {
if (data[index++] > hunks[h].memsize*4-4) {
printf("\nError in reloc table!\n");
return false;
}
}
}
hunks[h].relocentries = tot;
printf(" %6d%s\n", tot, note);
note = "";
missing_relocs = 0;
}
break;
case HUNK_END:
if (hunks[h].type == 0) {
printf("Empty%9d\n", hunks[h].memsize*4);
return false;
}
h = h+1; nh = h;
break;
case HUNK_RELOC16:
case HUNK_RELOC8:
case HUNK_EXT:
case HUNK_HEADER:
case HUNK_OVERLAY:
case HUNK_BREAK:
case HUNK_DREL32:
case HUNK_DREL16:
case HUNK_DREL8:
case HUNK_LIB:
case HUNK_INDEX:
case HUNK_RELOC32SHORT:
case HUNK_RELRELOC32:
case HUNK_ABSRELOC16:
printf("%s (unsupported)\n",hunktype[type-HUNK_UNIT]);
return false;
default:
printf("Unknown (%08X)\n",type);
return false;
}
}
}
if (index < length) {
printf("Warning: %d bytes of extra data at the end of the file!\n", (length-index)*4);
}
printf("\n");
return true;
}
int memory_usage(bool include_last_hunk) {
int sum = 0;
int hunks_to_sum = include_last_hunk ? hunks.size() : hunks.size() - 1;
for (int h = 0 ; h < hunks_to_sum ; h++) {
sum += ((hunks[h].memsize * 4 + 4) & -8) + 8;
}
return sum;
}
vector<pair<unsigned, vector<int> > > merged_hunklist() {
int numhunks = hunks.size();
vector<pair<unsigned, vector<int> > > hunklist(3);
unsigned flags0 = hunks[0].flags;
unsigned flags1 = (~flags0) & HUNKF_CHIP;
unsigned flags2 = HUNKF_CHIP + HUNKF_FAST - flags0 - flags1;
hunklist[0].first = flags0 | HUNK_CODE;
hunklist[1].first = flags1 | HUNK_CODE;
hunklist[2].first = flags2 | HUNK_CODE;
HunkMergeCompare comp(hunks);
// Go through the 3 resulting hunks, one for each memory type.
for (int dh = 0 ; dh < 3 ; dh++) {
for (int sh = 0 ; sh < numhunks ; sh++) {
if (hunks[sh].flags == (hunklist[dh].first & HUNKF_MASK)) {
hunklist[dh].second.push_back(sh);
}
}
stable_sort(hunklist[dh].second.begin(), hunklist[dh].second.end(), comp);
}
// Remove unused memory types
vector<pair<unsigned, vector<int> > > result;
for (int dh = 0 ; dh < 3 ; dh++) {
if (hunklist[dh].second.size() > 0) {
result.push_back(hunklist[dh]);
}
}
return result;
}
HunkFile* merge_hunks(const vector<pair<unsigned, vector<int> > >& hunklist) {
int numhunks = hunks.size();
int dnh = hunklist.size();
int bufsize = data.size()+3; // Reloc can write 3 further temporarily.
// Calculate safe size of new file buffer
for (int dh = 0 ; dh < dnh ; dh++) {
int waste = 0;
for (int shi = 0 ; shi < hunklist[dh].second.size() ; shi++) {
int sh = hunklist[dh].second[shi];
if (hunks[sh].type != HUNK_BSS) {
bufsize += waste;
waste = -hunks[sh].datasize;
}
waste += hunks[sh].memsize;
}
}
// Processed file
HunkFile *ef = new HunkFile;
ef->data.resize(bufsize, 0);
ef->hunks.resize(dnh);
vector<int> dhunk(numhunks);
vector<int> offset(numhunks);
// Find destination hunk and offset for all source hunks.
for (int dh = 0 ; dh < dnh ; dh++) {
unsigned hunkf = hunklist[dh].first;
ef->hunks[dh].type = hunkf & 0x0fffffff;
ef->hunks[dh].flags = hunkf & 0xf0000000;
int memsize = 0;
int datasize = 0;
for (int shi = 0 ; shi < hunklist[dh].second.size() ; shi++) {
int sh = hunklist[dh].second[shi];
memsize = (memsize+1)&-2;
dhunk[sh] = dh;
offset[sh] = memsize*4;
if (hunks[sh].type != HUNK_BSS) {
datasize = memsize + hunks[sh].datasize;
}
memsize += hunks[sh].memsize;
}
ef->hunks[dh].memsize = memsize;
ef->hunks[dh].datasize = datasize;
}
// Write new hunk header
int dpos = 0;
ef->data[dpos++] = HUNK_HEADER;
ef->data[dpos++] = 0;
ef->data[dpos++] = ef->hunks.size();
ef->data[dpos++] = 0;
ef->data[dpos++] = ef->hunks.size()-1;
for (int dh = 0 ; dh < ef->hunks.size() ; dh++) {
ef->data[dpos++] = ef->hunks[dh].memsize | ef->hunks[dh].flags;
}
// Generate new hunks
for (int dh = 0 ; dh < dnh ; dh++) {
// Put hunk type and data (or bss) size.
ef->data[dpos++] = ef->hunks[dh].type;
ef->data[dpos++] = ef->hunks[dh].datasize;
ef->hunks[dh].datastart = dpos;
// Run through the implied source hunks.
int hoffset = 0;
for (int shi = 0 ; shi < hunklist[dh].second.size() ; shi++) {
int sh = hunklist[dh].second[shi];
if (hunks[sh].type != HUNK_BSS) {
// Fill the gap.
for(; hoffset < offset[sh] ; hoffset += 4) {
ef->data[dpos++] = 0;
}
// Copy the data.
for (int spos = hunks[sh].datastart ; spos < hunks[sh].datastart + hunks[sh].datasize ; spos++) {
ef->data[dpos++] = data[spos];
}
hoffset += hunks[sh].datasize*4;
}
}
// Transfer all reloc information to the new hunk.
ef->data[dpos++] = HUNK_RELOC32;
ef->hunks[dh].relocstart = dpos;
ef->hunks[dh].relocentries = 0;
unsigned char *bytes = (unsigned char *)&ef->data[ef->hunks[dh].datastart];
// Iterate through destination reloc target hunk
for (int drh = 0 ; drh < ef->hunks.size() ; drh++) {
// Make space for number of relocs and store index of target hunk.
int rnpos = dpos++; // Position for number of relocs
ef->data[dpos++] = drh;
// Transfer all appropriate reloc entries.
int rtot = 0; // Total number of relocs in hunk
for (int sh = 0 ; sh < numhunks ; sh++) {
if (dhunk[sh] == dh && hunks[sh].relocentries > 0) {
int spos = hunks[sh].relocstart;
int rn; // Number of relocs
while ((rn = data[spos++]) > 0) {
int srh = data[spos++]; // Source reloc target hunk
if (dhunk[srh] == drh) {
rtot += rn;
for (int ri = 0 ; ri < rn ; ri++) {
int rv = data[spos++]; // Reloc value
ef->data[dpos++] = rv+offset[sh];
*((Longword *)&bytes[rv+offset[sh]]) += offset[srh];
}
} else {
spos += rn;
}
}
}
}
// Store total number of relocs with the actual target hunk.
// If there are none, remove the spaces for
// number of relocs and target hunk.
if (rtot == 0) {
dpos -= 2;
} else {
ef->data[rnpos] = rtot;
ef->hunks[dh].relocentries += rtot;
}
}
// End the reloc section.
// If there are no relocs, remove the reloc header.
if (ef->hunks[dh].relocentries == 0) {
dpos -= 1;
} else {
ef->data[dpos++] = 0;
}
}
// There must be a HUNK_END after last hunk!
ef->data[dpos++] = HUNK_END;
// Note resulting file size
ef->data.resize(dpos);
return ef;
}
bool valid_mini() {
if (!(hunks[0].type == HUNK_CODE && hunks[0].relocentries == 0)) return false;
for (int h = 1 ; h < hunks.size() ; h++) {
if (hunks[h].relocentries != 0) return false;
if (hunks[h].type == HUNK_BSS || hunks[h].datasize == 0) continue;
for (int i = 0 ; i < hunks[h].datasize ; i++) {
if (data[hunks[h].datastart + i] != 0) return false;
}
}
return true;
}
HunkFile* crunch(PackParams *params, bool overlap, bool mini, string *decrunch_text, unsigned flash_address, RefEdgeFactory *edge_factory, bool show_progress) {
vector<unsigned> pack_buffer = compress_hunks(params, overlap, mini, edge_factory, show_progress);
vector<pair<int,int> > count_and_hunksize = verify(pack_buffer, overlap, mini);
int numhunks = hunks.size();
int newnumhunks = numhunks+1;
int bufsize = data.size() * 11 / 10 + 1000;
HunkFile *ef = new HunkFile;
ef->data.resize(bufsize, 0);
int dpos = 0;
// Write new hunk header
ef->data[dpos++] = HUNK_HEADER;
ef->data[dpos++] = 0;
ef->data[dpos++] = newnumhunks;
ef->data[dpos++] = 0;
ef->data[dpos++] = newnumhunks-1;
int lpos1, lpos2, ppos;
Word *offsetp = NULL;
if (overlap) {
// Write hunk memory sizes
lpos1 = dpos++;
for (int h = 0 ; h < numhunks ; h++) {
int hunksize = max(hunks[h].memsize, count_and_hunksize[h].second);
ef->data[dpos++] = hunksize | hunks[h].flags;
}
// Write header
ef->data[dpos++] = HUNK_CODE;
lpos2 = dpos++;
ppos = dpos;
if (decrunch_text) {
memcpy(&ef->data[dpos], OverlapHeaderT, sizeof(OverlapHeaderT));
dpos += sizeof(OverlapHeaderT) / sizeof(Longword);
ef->data[ppos + 4] = decrunch_text->length();
offsetp = (Word *) &ef->data[ppos + 10];
} else {
memcpy(&ef->data[dpos], OverlapHeader, sizeof(OverlapHeader));
dpos += sizeof(OverlapHeader) / sizeof(Longword);
}
} else if (mini) {
// Write hunk memory sizes
lpos1 = dpos++;
for (int h = 0 ; h < numhunks ; h++) {
ef->data[dpos++] = hunks[h].memsize | hunks[h].flags;
}
// Write header
ef->data[dpos++] = HUNK_CODE;
lpos2 = dpos++;
ppos = dpos;
memcpy(&ef->data[dpos], MiniHeader, sizeof(MiniHeader));
dpos += sizeof(MiniHeader) / sizeof(Longword);
offsetp = (Word *) (((unsigned char *) &ef->data[ppos]) + 12);
} else {
int header1_size = sizeof(Header1) / sizeof(Longword);
if (decrunch_text) {
header1_size = (sizeof(Header1T) + (decrunch_text->length() + 3)) / sizeof(Longword);
}
for (int h = 0 ; h < numhunks ; h++) {
int memsize = hunks[h].memsize;
if (h == 0 && memsize < header1_size) {
// Make space for header trampoline code
memsize = header1_size;
}
ef->data[dpos++] = memsize | hunks[h].flags;
}
lpos1 = dpos++;
// Write header 1
ef->data[dpos++] = HUNK_CODE;
ef->data[dpos++] = header1_size;
if (decrunch_text) {
memset(&ef->data[dpos], 0, header1_size);
memcpy(&ef->data[dpos], Header1T, sizeof(Header1T));
char *text_dest = ((char *) &ef->data[dpos]) + sizeof(Header1T);
memcpy(text_dest, decrunch_text->c_str(), decrunch_text->length());
ef->data[dpos + 5] = decrunch_text->length();
} else {
memcpy(&ef->data[dpos], Header1, sizeof(Header1));
}
dpos += header1_size;
// Write hunks
for (int h = 1 ; h < numhunks ; h++) {
ef->data[dpos++] = hunks[h].type;
switch (hunks[h].type) {
case HUNK_CODE:
case HUNK_DATA:
ef->data[dpos++] = 0;
break;
case HUNK_BSS:
ef->data[dpos++] = hunks[h].datasize;
break;
}
}
// Write header 2
ef->data[dpos++] = HUNK_CODE;
lpos2 = dpos++;
ppos = dpos;
memcpy(&ef->data[dpos], Header2, sizeof(Header2));
dpos += sizeof(Header2) / sizeof(Longword);
offsetp = (Word *) (((unsigned char *) &ef->data[ppos]) + 4);
}
if (flash_address) {
// Insert flashing code
dpos += 1;
for (int fpos = dpos - 1 ; fpos >= dpos - 9 ; fpos--) {
ef->data[fpos] = ef->data[fpos - 1];
}
Word* insts = (Word *) &ef->data[dpos - 11];
insts[0] = 0x33C3; // move.w d3,flash_address
*(Longword *)&insts[1] = flash_address;
insts[3] = 0x6AEC; // bpl.b readbit
if (offsetp) *offsetp += 4;
}
if (overlap) {
// Write decrunch text
if (decrunch_text) {
int rounded_text_size = (decrunch_text->length() + 3) & -4;
memset(&ef->data[dpos], 0, rounded_text_size);
memcpy(&ef->data[dpos], decrunch_text->c_str(), decrunch_text->length());
dpos += rounded_text_size / sizeof(Longword);
}
// Set hunk sizes
ef->data[lpos1] = dpos-ppos;
ef->data[lpos2] = dpos-ppos;
// Write hunks
int packed_index = 0;
for (int h = 0 ; h < numhunks ; h++) {
ef->data[dpos++] = HUNK_DATA;
int longwords_in_hunk = min<int>(count_and_hunksize[h].first, pack_buffer.size() - packed_index);
ef->data[dpos++] = longwords_in_hunk + 1;
ef->data[dpos++] = count_and_hunksize[h].first * 4;
for (int i = 0 ; i < longwords_in_hunk ; i++) {
ef->data[dpos++] = pack_buffer[packed_index++];
}
}
} else if (mini) {
// Write compressed data backwards
for (int i = pack_buffer.size()-1 ; i >= 0 ; i--) {
ef->data[dpos++] = pack_buffer[i];
}
// Set hunk sizes
ef->data[lpos1] = dpos-ppos + 32768/8*2/4; // Space for context state
ef->data[lpos2] = dpos-ppos;
// Write hunks
for (int h = 0 ; h < numhunks ; h++) {
ef->data[dpos++] = HUNK_BSS;
ef->data[dpos++] = hunks[h].memsize;
}
// Set size of data in header
int offset = (int) *offsetp + pack_buffer.size() * 4;
if (offset > 32767) {
printf("Size overflow: final size in mini mode must be less than 24k.\n\n");
exit(1);
}
*offsetp = offset;
} else {
// Write compressed data
for (int i = 0 ; i < pack_buffer.size() ; i++) {
ef->data[dpos++] = pack_buffer[i];
}
// Set hunk sizes
ef->data[lpos1] = dpos-ppos + 1; // Space for range decoder overshoot
ef->data[lpos2] = dpos-ppos;
}
// There must be a HUNK_END after last hunk!
ef->data[dpos++] = HUNK_END;
// Note resulting file size
ef->data.resize(dpos);
return ef;
}
};
+35
View File
@@ -0,0 +1,35 @@
Shrinkler executable file compressor for Amiga
Copyright 1999-2015 Aske Simon Christensen, with exceptions noted below.
Permission is hereby granted to anyone obtaining a copy of this software
package (including accompanying documentation) to compile, use, copy,
modify, merge and/or distribute it, in whole or in part, subject to the
following conditions:
- Distribution in source code form must include a copy of this license.
- Distribution in binary form must not be misattributed, i.e. you must
not claim (implicitly or explicitly) that you wrote it yourself.
- Distribution of the decrunch headers (Header.S, MiniHeader.S,
OverlapHeader.S, and the .bin and .dat files generated from them) in
binary form as part of an Amiga executable is not restricted by this
license and does not require attribution.
In particular, output executables from Shrinkler (which contain code
from the decrunch headers) are to be considered original works of the
author(s) of the corresponding input executables.
- The data decompression code (ShrinklerDecompress.S) is distributed
alongside the Shrinkler binaries in the official archives and has its
own license stated inside the file.
Exceptions:
- doshunks.h is part of the Amiga SDK and is Copyright 1989-1993
Commodore-Amiga, Inc.
- The "as" and "ld" executables in the toolchain directory are taken
from the AmiDevCpp distribution. They are covered by the GNU General
Public License.
+74
View File
@@ -0,0 +1,74 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
Decoder for the LZ encoder.
*/
#pragma once
#include "Decoder.h"
#include "LZEncoder.h"
class LZReceiver {
public:
virtual bool receiveLiteral(unsigned char value) = 0;
virtual bool receiveReference(int offset, int length) = 0;
virtual ~LZReceiver() {}
};
class LZDecoder {
Decoder *decoder;
int decode(int context) const {
return decoder->decode(LZEncoder::NUM_SINGLE_CONTEXTS + context);
}
int decodeNumber(int context_group) const {
return decoder->decodeNumber(LZEncoder::NUM_SINGLE_CONTEXTS + (context_group << 8));
}
public:
LZDecoder(Decoder *decoder) : decoder(decoder) {
}
bool decode(LZReceiver& receiver) {
bool ref = false;
bool prev_was_ref = false;
int pos = 0;
int offset = 0;
do {
if (ref) {
bool repeated = false;
if (!prev_was_ref) {
repeated = decode(LZEncoder::CONTEXT_REPEATED);
}
if (!repeated) {
offset = decodeNumber(LZEncoder::CONTEXT_GROUP_OFFSET) - 2;
if (offset == 0) break;
}
int length = decodeNumber(LZEncoder::CONTEXT_GROUP_LENGTH);
if (!receiver.receiveReference(offset, length)) return false;
pos += length;
prev_was_ref = true;
} else {
int parity = pos & 1;
int context = 1;
for (int i = 7 ; i >= 0 ; i--) {
int bit = decode((parity << 8) | context);
context = (context << 1) | bit;
}
unsigned char lit = context;
if (!receiver.receiveLiteral(lit)) return false;
pos += 1;
prev_was_ref = false;
}
int parity = pos & 1;
ref = decode(LZEncoder::CONTEXT_KIND + (parity << 8));
} while (true);
return true;
}
};
+182
View File
@@ -0,0 +1,182 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
The LZ encoder defines the encoding of LZ symbols (literal bytes and references) into data bytes.
The encoding consists of three layers:
Layer 1 defines a plain encoding into bits. It is as follows:
The first synbol is always a literal, and it is encoded as
bit7 .. bit0
Subsequent symbols can be either literals or references, and are encoded as one of
0 bit7 .. bit0 (literal byte)
1 0 <offset+2> <length> (reference)
1 1 <length> (reference with same offset as previous reference)
and the data block for each hunk is terminated by
1 0 <2>
The data block is followed by relocation entries, specifying positions within the data where the
address of some hunk must be added. The entries are separated into one list for each hunk.
Within each list, each entry is encoded as
<delta from previous position> (the position before the first entry is assumed to be -4)
and each list is terminated by
<2>
The <number> encodings in the above are variable-length numbers with a value of 2 or greater.
The number 1 bit(n-1) .. bit0 is encoded as
1^(n-1) 0 bit(n-1) .. bit0
Layer 2 defines a context for each bit of the Layer 1 encoding. The probability distribution
between 0 and 1 is modelled adaptively for each context.
The first bit of the general symbol encoding (the one that selects between literal and reference)
has one context for each parity of the byte position in the data (i.e. one for even bytes and one
for odd bytes).
The second bit of the reference symbol encoding (the one that selects between new and repeated
offset) has a single context for itself.
Literal bits have one context for each combination of parity and all higher numbered bits within
the same literal byte. Thus, there are 510 different literal contexts.
Numbers have one context group for each of offset, length and relocation entry. Within each group,
there is one context for each of the prefix bits, and one context for each data bit number (i.e.
bit(i) always uses the same context for all numbers with more than i data bits).
Layer 3 performs entropy coding of the Layer 1 bits based on the probabilities estimated by Layer 2.
The entropy coder defines the final compressed data contents.
*/
#pragma once
#include "Coder.h"
class LZState {
unsigned after_first:1;
unsigned prev_was_ref:1;
unsigned parity:1;
unsigned last_offset:28;
friend class LZEncoder;
};
class LZEncoder {
static const int NUM_SINGLE_CONTEXTS = 1;
static const int NUM_CONTEXT_GROUPS = 4;
static const int CONTEXT_GROUP_SIZE = 256;
static const int CONTEXT_KIND = 0;
static const int CONTEXT_REPEATED = -1;
static const int CONTEXT_GROUP_LIT = 0;
static const int CONTEXT_GROUP_OFFSET = 2;
static const int CONTEXT_GROUP_LENGTH = 3;
Coder *coder;
int code(int context, int bit) const {
return coder->code(NUM_SINGLE_CONTEXTS + context, bit);
}
int encodeNumber(int context_group, int number) const {
return coder->encodeNumber(NUM_SINGLE_CONTEXTS + (context_group << 8), number);
}
friend class LZDecoder;
public:
static const int KIND_LIT = 0;
static const int KIND_REF = 1;
static const int NUM_CONTEXTS = (NUM_SINGLE_CONTEXTS + NUM_CONTEXT_GROUPS * CONTEXT_GROUP_SIZE);
static const int NUMBER_CONTEXT_OFFSET = (NUM_SINGLE_CONTEXTS + CONTEXT_GROUP_OFFSET * CONTEXT_GROUP_SIZE);
static const int NUM_NUMBER_CONTEXTS = 2;
LZEncoder(Coder *coder) : coder(coder) {
}
void setInitialState(LZState *state) const {
state->after_first = 0;
state->prev_was_ref = 0;
state->parity = 0;
state->last_offset = 0;
}
void constructState(LZState *state, int pos, bool prev_was_ref, int last_offset) const {
state->after_first = pos > 0;
state->prev_was_ref = prev_was_ref;
state->parity = pos;
state->last_offset = last_offset;
}
int encodeLiteral(unsigned char value, const LZState *state_before, LZState *state_after) const {
int size = 0;
if (state_before->after_first) {
size += code(CONTEXT_KIND + (state_before->parity << 8), KIND_LIT);
}
int context = 1;
for (int i = 7 ; i >= 0 ; i--) {
int bit = ((value >> i) & 1);
size += code((state_before->parity << 8) | context, bit);
context = (context << 1) | bit;
}
state_after->after_first = 1;
state_after->prev_was_ref = 0;
state_after->parity = state_before->parity + 1;
state_after->last_offset = state_before->last_offset;
return size;
}
int encodeReference(int offset, int length, const LZState *state_before, LZState *state_after) const {
assert(offset >= 1);
assert(length >= 2);
assert(state_before->after_first);
int size = code(CONTEXT_KIND + (state_before->parity << 8), KIND_REF);
int rep_offset = offset == state_before->last_offset;
if (!state_before->prev_was_ref) {
size += code(CONTEXT_REPEATED, rep_offset);
} else {
assert(!rep_offset);
}
if (!rep_offset) {
size += encodeNumber(CONTEXT_GROUP_OFFSET, offset + 2);
}
size += encodeNumber(CONTEXT_GROUP_LENGTH, length);
state_after->after_first = 1;
state_after->prev_was_ref = 1;
state_after->parity = state_before->parity + length;
state_after->last_offset = offset;
return size;
}
int finish(const LZState *state_before) const {
int size = code(CONTEXT_KIND + (state_before->parity << 8), KIND_REF);
if (!state_before->prev_was_ref) {
size += code(CONTEXT_REPEATED, 0);
}
int context_group = CONTEXT_GROUP_OFFSET;
int number = 2;
size += encodeNumber(context_group, number);
return size;
}
};
+408
View File
@@ -0,0 +1,408 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
Parse a data block into LZ symbols (literal bytes and references).
The parser uses a "local optimal parse" strategy, where all matches reported
by the match finder are considered. Potential parses are maintained for each
possible previous reference offset, in order to maximize the utilization of
the "repeated offset" feature of the LZ encoding.
Three parameters control the speed/precision tradeoff of the parser:
The length_margin parameter how many shorter matches the parser will consider
for each match reported by the match finder. If the match finder reports a
match of length l, the parser will consider all (valid) matches of length at
least l-length_margin.
The skip_length parameter controls a shortcutting mechanism for very long
matches. Whenever a match of length at least skip_length is reported, the
parser will use that match unconditionally and skip ahead to continue the
parsing at the end of the match.
The max_edges parameter controls the total number of reference edges the
parser will keep around for representing potential parses. Whenever the
limit is reached, the parser will delete the least favorable of the current
parses to free up space.
*/
#pragma once
#include <vector>
#include <map>
#include <set>
#include <functional>
#include <utility>
#include <list>
#include <algorithm>
using std::map;
using std::max;
using std::min;
using std::pair;
using std::sort;
using std::vector;
#include "LZEncoder.h"
#include "MatchFinder.h"
#include "Heap.h"
#include "CuckooHash.h"
#include "assert.h"
// For each offset:
// Best total size with last ref having that offset
class RefEdge {
int pos;
int offset;
int length;
int total_size;
int refcount;
RefEdge *source;
RefEdge(int pos, int offset, int length, int total_size, RefEdge *source)
: pos(pos), offset(offset), length(length), total_size(total_size), source(source)
{
assert(source != this);
refcount = 1;
if (source != NULL) {
source->refcount++;
}
}
int target() {
return pos + length;
}
friend class RefEdgeFactory;
friend class LZParser;
friend class LZResultEdge;
friend class LZParseResult;
friend struct std::less<RefEdge*>;
public:
int _heap_index;
};
namespace std {
template <> struct less<RefEdge*> {
bool operator()(RefEdge* const & e1, RefEdge* const & e2) const {
return e1->total_size < e2->total_size;
}
};
}
// Factory for RefEdge objects which recycles destroyed objects for efficiency
class RefEdgeFactory {
int edge_capacity;
int edge_count;
int cleaned_edges;
RefEdge* buffer;
public:
int max_edge_count;
int max_cleaned_edges;
RefEdgeFactory(int edge_capacity) : edge_capacity(edge_capacity),
edge_count(0), cleaned_edges(0), max_edge_count(0), max_cleaned_edges(0)
{
buffer = NULL;
}
~RefEdgeFactory() {
while (buffer != NULL) {
RefEdge *edge = buffer;
buffer = buffer->source;
delete edge;
}
}
void reset() {
assert(edge_count == 0);
cleaned_edges = 0;
}
RefEdge* create(int pos, int offset, int length, int total_size, RefEdge *source) {
max_edge_count = max(max_edge_count, ++edge_count);
if (buffer == NULL) {
return new RefEdge(pos, offset, length, total_size, source);
} else {
RefEdge* edge = buffer;
buffer = edge->source;
return new (edge) RefEdge(pos, offset, length, total_size, source);
}
}
void destroy(RefEdge* edge, bool clean) {
edge->source = buffer;
buffer = edge;
edge_count--;
if (clean) {
max_cleaned_edges = max(max_cleaned_edges, ++cleaned_edges);
}
}
bool full() {
return edge_count >= edge_capacity;
}
};
class LZProgress {
public:
virtual void begin(int size) = 0;
virtual void update(int pos) = 0;
virtual void end() = 0;
virtual ~LZProgress() {}
};
struct LZResultEdge {
int pos;
int offset;
int length;
LZResultEdge(RefEdge *edge) : pos(edge->pos), offset(edge->offset), length(edge->length) {}
friend class LZParseResult;
};
class LZParseResult {
vector<LZResultEdge> edges;
const unsigned char *data;
int data_length;
int zero_padding;
public:
int encode(const LZEncoder& result_encoder) const {
int size = 0;
int pos = 0;
LZState state;
result_encoder.setInitialState(&state);
for (int i = edges.size() - 1 ; i >= 0 ; i--) {
const LZResultEdge *edge = &edges[i];
while (pos < edge->pos) {
size += result_encoder.encodeLiteral(data[pos++], &state, &state);
}
size += result_encoder.encodeReference(edge->offset, edge->length, &state, &state);
pos += edge->length;
}
while (pos < data_length) {
size += result_encoder.encodeLiteral(data[pos++], &state, &state);
}
if (zero_padding > 0) {
size += result_encoder.encodeLiteral(0, &state, &state);
if (zero_padding == 2) {
size += result_encoder.encodeLiteral(0, &state, &state);
} else if (zero_padding > 1) {
size += result_encoder.encodeReference(1, zero_padding - 1, &state, &state);
}
}
size += result_encoder.finish(&state);
return size;
}
friend class LZParser;
};
class LZParser {
const unsigned char *data;
int data_length;
int zero_padding;
MatchFinder& finder;
int length_margin;
int skip_length;
const LZEncoder* encoderp;
RefEdgeFactory* edge_factory;
vector<int> literal_size;
vector<CuckooHash<RefEdge*> > edges_to_pos;
RefEdge* best;
CuckooHash<RefEdge*> best_for_offset;
Heap<RefEdge*> root_edges;
bool is_root(RefEdge *edge) {
return root_edges.contains(edge);
}
void remove_root(RefEdge *edge) {
root_edges.remove(edge);
}
void releaseEdge(RefEdge *edge, bool clean = false) {
while (edge != NULL) {
RefEdge *source = edge->source;
if (--edge->refcount == 0) {
assert(!is_root(edge));
edge_factory->destroy(edge, clean);
} else {
return;
}
edge = source;
}
}
// Return progress
bool clean_worst_edge(int pos, RefEdge *exclude) {
if (root_edges.size() == 0) return false;
RefEdge *worst_edge = root_edges.remove_largest();
if (worst_edge == best || worst_edge == exclude) return true;
CuckooHash<RefEdge*>& container = worst_edge->target() > pos
? edges_to_pos[worst_edge->target()]
: best_for_offset;
if (container.size() > 1 && container.count(worst_edge->offset) > 0) {
container.erase(worst_edge->offset);
releaseEdge(worst_edge, true);
}
return true;
}
void put_by_offset(CuckooHash<RefEdge*>& by_offset, RefEdge* edge) {
assert(!is_root(edge));
if (by_offset.count(edge->offset) == 0) {
by_offset[edge->offset] = edge;
root_edges.insert(edge);
} else if (edge->total_size < by_offset[edge->offset]->total_size) {
RefEdge* old_edge = by_offset[edge->offset];
remove_root(old_edge);
releaseEdge(old_edge);
by_offset[edge->offset] = edge;
root_edges.insert(edge);
} else {
releaseEdge(edge);
}
}
void newEdge(RefEdge *source, int pos, int offset, int length) {
if (source && offset == source->offset && pos == source->target()) return;
int prev_target = source ? source->target() : 0;
int new_target = pos + length;
LZState state_before;
LZState state_after;
encoderp->constructState(&state_before, pos, pos == prev_target, source ? source->offset : 0);
int size_before = (source ? source->total_size : literal_size[data_length]) - (literal_size[data_length] - literal_size[pos]);
int edge_size = encoderp->encodeReference(offset, length, &state_before, &state_after);
int size_after = literal_size[data_length] - literal_size[new_target];
while (edge_factory->full()) {
if (!clean_worst_edge(pos, source)) break;
}
RefEdge *new_edge = edge_factory->create(pos, offset, length, size_before + edge_size + size_after, source);
put_by_offset(edges_to_pos[new_target], new_edge);
}
public:
LZParser(const unsigned char *data, int data_length, int zero_padding, MatchFinder& finder, int length_margin, int skip_length, RefEdgeFactory* edge_factory)
: data(data), data_length(data_length), zero_padding(zero_padding), finder(finder), length_margin(length_margin), skip_length(skip_length), edge_factory(edge_factory)
{
// Initialize edges_to_pos array
edges_to_pos.resize(data_length + 1);
best = NULL;
}
LZParseResult parse(const LZEncoder& encoder, LZProgress *progress) {
progress->begin(data_length);
encoderp = &encoder;
// Reset state
best_for_offset.clear();
root_edges.clear();
edge_factory->reset();
// Accumulate literal sizes
literal_size.resize(data_length + 1, 0);
int size = 0;
LZState literal_state;
encoder.setInitialState(&literal_state);
for (int i = 0 ; i < data_length ; i++) {
literal_size[i] = size;
size += encoder.encodeLiteral(data[i], &literal_state, &literal_state);
}
literal_size[data_length] = size;
// Parse
RefEdge* initial_best = edge_factory->create(0, 0, 0, literal_size[data_length], NULL);
best = initial_best;
for (int pos = 1 ; pos <= data_length ; pos++) {
// Assimilate edges ending here
for (CuckooHash<RefEdge*>::iterator it = edges_to_pos[pos].begin() ; it != edges_to_pos[pos].end() ; it++) {
RefEdge *edge = it->second;
if (edge->total_size < best->total_size) {
best = edge;
}
remove_root(edge);
put_by_offset(best_for_offset, edge);
}
edges_to_pos[pos].clear();
// Add new edges according to matches
finder.beginMatching(pos);
int match_pos;
int match_length;
int max_match_length = 0;
while (finder.nextMatch(&match_pos, &match_length)) {
int offset = pos - match_pos;
if (match_length > data_length - pos) {
match_length = data_length - pos;
}
int min_length = match_length - length_margin;
if (min_length < 2) min_length = 2;
for (int length = min_length ; length <= match_length ; length++) {
newEdge(best, pos, offset, length);
if (best->offset != offset && best_for_offset.count(offset)) {
assert(best_for_offset[offset]->target() <= pos);
newEdge(best_for_offset[offset], pos, offset, length);
}
}
max_match_length = max(max_match_length, match_length);
}
// If we have a very long match, skip ahead
if (max_match_length >= skip_length && !edges_to_pos[pos + max_match_length].empty()) {
root_edges.clear();
for (CuckooHash<RefEdge*>::iterator it = best_for_offset.begin() ; it != best_for_offset.end() ; it++) {
releaseEdge(it->second);
}
best_for_offset.clear();
int target_pos = pos + max_match_length;
while (pos < target_pos - 1) {
CuckooHash<RefEdge*>& edges = edges_to_pos[++pos];
for (CuckooHash<RefEdge*>::iterator it = edges.begin() ; it != edges.end() ; it++) {
releaseEdge(it->second);
}
edges.clear();
}
best = initial_best;
}
progress->update(pos);
}
// Clean unused paths
root_edges.clear();
for (CuckooHash<RefEdge*>::iterator it = best_for_offset.begin() ; it != best_for_offset.end() ; it++) {
RefEdge *edge = it->second;
if (edge != best) {
releaseEdge(edge);
}
}
// Find best path
LZParseResult result;
result.data = data;
result.data_length = data_length;
result.zero_padding = zero_padding;
RefEdge *edge = best;
while (edge->length > 0) {
result.edges.push_back(LZResultEdge(edge));
edge = edge->source;
}
releaseEdge(edge);
releaseEdge(best);
progress->end();
return result;
}
};
+115
View File
@@ -0,0 +1,115 @@
ifndef PLATFORM
PLATFORM := native
endif
ifneq ($(PLATFORM),$(filter $(PLATFORM),amiga windows-32 windows-64 native native-32 native-64))
DUMMY := $(error Unsupported platform $(PLATFORM))
endif
BUILD_DIR := build/$(PLATFORM)
MKDIR_DUMMY := $(shell mkdir -p $(BUILD_DIR))
all: $(BUILD_DIR)/Shrinkler
# Common flags
CFLAGS := -Wall -Wno-sign-compare
LFLAGS := -s
ifdef DEBUG
CFLAGS += -g -DDEBUG
LFLAGS :=
else
CFLAGS += -O3
endif
ifdef PROFILE
CFLAGS += -fno-inline -fno-inline-functions
LFLAGS :=
endif
ifeq ($(PLATFORM),amiga)
# Amiga build, using GCC and ixemul
TOOLCHAIN_DIR := toolchain
AMIGA_GCC_DIR := $(TOOLCHAIN_DIR)/GCC-4.5.0-m68k-amigaos-cygwin/usr/local/amiga
BINUTILS_DIR := $(TOOLCHAIN_DIR)/amiga-binutils
INCLUDE_DIR := $(TOOLCHAIN_DIR)/C++include/include
LIB_DIR1 := $(TOOLCHAIN_DIR)/C++include/lib
LIB_DIR2 := $(AMIGA_GCC_DIR)/lib/gcc/m68k-amigaos/4.5.0
LIB_DIR3 := $(TOOLCHAIN_DIR)/ixemul-sdk/lib
CC := $(AMIGA_GCC_DIR)/bin/m68k-amigaos-g++
CFLAGS += -m68000
INCLUDE := -I $(INCLUDE_DIR)/c++/4.3.2 -I $(INCLUDE_DIR)/c++/4.3.2/m68k-amigaos -I $(INCLUDE_DIR)
ASM := $(BINUTILS_DIR)/as
ASMFLAGS :=
LINK := $(BINUTILS_DIR)/ld
STARTUP := $(LIB_DIR3)/crt0.o
LIBS := -L $(LIB_DIR1) -L $(LIB_DIR2) -L $(LIB_DIR3) -lstdc++ -lgcc -lc
$(BUILD_DIR)/%.o: %.cpp
$(CC) $(CFLAGS) $(INCLUDE) $< -S -o $(@:%.o=%.s)
$(ASM) $(ASMFLAGS) $(@:%.o=%.s) -o $@
else
ifeq ($(PLATFORM),windows-32)
# 32-bit MinGW build
CC := i686-w64-mingw32-g++
LINK := i686-w64-mingw32-g++
LFLAGS += -static-libgcc -static-libstdc++
else
ifeq ($(PLATFORM),windows-64)
# 64-bit MinGW build
CC := x86_64-w64-mingw32-g++
LINK := x86_64-w64-mingw32-g++
LFLAGS += -static-libgcc -static-libstdc++
else
# Native build
CC := g++
LINK := g++
ifeq ($(PLATFORM),native-32)
CFLAGS += -m32
endif
ifeq ($(PLATFORM),native-64)
CFLAGS += -m64
endif
endif
endif
# Common setup for non-Amiga builds
INCLUDE :=
STARTUP :=
LIBS :=
$(BUILD_DIR)/%.o: %.cpp
$(CC) $(CFLAGS) $(INCLUDE) $< -c -o $@
endif
$(BUILD_DIR)/Shrinkler.o: *.h Header1.dat Header1T.dat Header2.dat OverlapHeader.dat OverlapHeaderT.dat MiniHeader.dat
%.dat: %.bin
python -c 'for b in open("$^", "rb").read(): print ("0x%02X," % ord(b)),' > $@
$(BUILD_DIR)/Shrinkler: $(BUILD_DIR)/Shrinkler.o
$(LINK) $(LFLAGS) $(STARTUP) $< $(LIBS) -o $@
clean:
rm -rf build
+211
View File
@@ -0,0 +1,211 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
Find repeated strings in a data block.
Matches are reported from longest to shortest. A match is only reported
if it is closer (smaller offset, higher position) than all longer matches.
Two parameters control the speed/precision tradeoff of the matcher:
The match_patience parameter controls how many matches outside the current
reporting range (between last longer match and current position) are skipped
before the matcher gives up finding more matches.
The max_same_length parameter controls how many matches of the same length
are reported. The matches reported will be the closest ones of that length.
*/
#pragma once
#include <vector>
#include <algorithm>
#include <queue>
#include <functional>
using std::vector;
class MatchFinder {
// Inputs
unsigned char *data;
int length;
int min_length;
int match_patience;
int max_same_length;
// Suffix array
vector<int> suffix_array;
vector<int> rev_suffix_array;
vector<int> longest_common_prefix;
// Matcher parameters
int current_pos;
int min_pos;
// Matcher state
int left_index;
int left_length;
int right_index;
int right_length;
int current_length;
// Best matches seen with current length
std::priority_queue<int, vector<int>, std::greater<int> > match_buffer;
struct suffix_compare {
MatchFinder* finder;
suffix_compare(MatchFinder* finder) : finder(finder) {}
// Compare suffixes starting at positions a and b
bool operator()(int a, int b) {
vector<int>& same = finder->rev_suffix_array;
unsigned char *data = finder->data;
if (a == b) return false;
if (data[a] == data[b]) {
// Skip stretch of equal bytes
int skip = std::min(same[a], same[b]);
a += skip;
b += skip;
}
int until_end = finder->length - std::max(a,b);
for (int i = 0 ; i < until_end ; i++) {
if (data[a + i] != data[b + i]) {
return data[a + i] < data[b + i];
}
}
return a < b;
}
};
// Quick'n'dirty suffix array construction:
// plain sorting with accelleration of same-value blocks
void make_suffix_array() {
// Temporary same-value block accelleration array
rev_suffix_array.resize(length + 1);
int count = 0;
char c = 0;
rev_suffix_array[length] = 0;
for (int i = length - 1 ; i >= 0 ; i--) {
if (data[i] != c) count = 0;
rev_suffix_array[i] = ++count;
c = data[i];
}
// Compute suffix array
suffix_array.resize(length + 1);
for (int i = 0 ; i <= length ; i++) {
suffix_array[i] = i;
}
std::sort(&suffix_array[0], &suffix_array[length], suffix_compare(this));
// Compute reverse suffix array
for (int i = 0 ; i <= length ; i++) {
rev_suffix_array[suffix_array[i]] = i;
}
// Compute LCP array
longest_common_prefix.resize(length + 1);
longest_common_prefix[0] = 0;
longest_common_prefix[length] = 0;
int h = 0;
for (int i = 0 ; i < length ; i++) {
int r = rev_suffix_array[i];
if (r > 0) {
int j = suffix_array[r - 1];
while (data[i + h] == data[j + h]) {
h = h + 1;
}
longest_common_prefix[r] = h;
if (h > 0) h = h - 1;
}
}
}
void extend_left() {
int iter = 0;
while (left_length >= min_length) {
left_length = std::min(left_length, longest_common_prefix[left_index]);
int pos = suffix_array[--left_index];
if (pos < current_pos && pos >= min_pos) break;
if (++iter > match_patience) left_length = 0;
}
}
void extend_right() {
int iter = 0;
while (right_length >= min_length) {
right_length = std::min(right_length, longest_common_prefix[++right_index]);
int pos = suffix_array[right_index];
if (pos < current_pos && pos >= min_pos) break;
if (++iter > match_patience) right_length = 0;
}
}
int next_length() {
return std::max(left_length, right_length);
}
public:
MatchFinder(unsigned char *data, int length, int min_length, int match_patience, int max_same_length) :
data(data), length(length), min_length(min_length), match_patience(match_patience), max_same_length(max_same_length) {
make_suffix_array();
reset();
}
void reset() {
}
// Start finding matches between strings starting at pos and earlier strings.
void beginMatching(int pos) {
current_pos = pos;
min_pos = 0;
left_index = rev_suffix_array[pos];
left_length = length;
extend_left();
right_index = rev_suffix_array[pos];
right_length = length;
extend_right();
}
// Report next match. Returns whether a match was found.
bool nextMatch(int *match_pos_out, int *match_length_out) {
if (match_buffer.empty()) {
// Fill match buffer
current_length = next_length();
if (current_length < min_length) return false;
int new_min_pos = min_pos;
do {
int match_pos;
if (left_length > right_length) {
match_pos = suffix_array[left_index];
extend_left();
} else {
match_pos = suffix_array[right_index];
extend_right();
}
new_min_pos = std::max(new_min_pos, match_pos);
if (match_buffer.size() < max_same_length) {
match_buffer.push(match_pos);
} else {
if (match_pos > match_buffer.top()) {
match_buffer.pop();
match_buffer.push(match_pos);
}
min_pos = match_buffer.top();
}
} while (next_length() == current_length);
assert(!match_buffer.empty());
min_pos = new_min_pos;
}
*match_length_out = current_length;
*match_pos_out = match_buffer.top();
match_buffer.pop();
assert(*match_pos_out < current_pos);
return true;
}
};
+156
View File
@@ -0,0 +1,156 @@
; Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
; auto wb\MiniHeader\MiniHeader_End\
INIT_ONE_PROB = $8000
ADJUST_SHIFT = 4
SINGLE_BIT_CONTEXTS = 1
DUMMY_CONTEXT_OFFSET = 0
LIB_VERSION = 20
CacheClearU = -636
align 0,4
MiniHeader:
move.l (a3),d2
lsl.l #2,d2
move.l d2,a1
addq.l #4,a1
pea.l (a1)
ContextOffsetInstr:
lea.l MiniHeader_End+DUMMY_CONTEXT_OFFSET+(32768/8)*2(pc),a2
moveq.l #0,d1
.init: move.w #INIT_ONE_PROB,-(a2)
addq.w #8,d1
bpl.b .init
move.l a2,a4
Depack:
; A4 = Packed data End
; A1 = Target
; A2 = Contexts
; D1 = $00008000
; Lowest bit of D2 = 0
swap.w d1
moveq.l #1,d3
moveq.l #0,d6
.lit:
addq.b #1,d6
.getlit:
bsr.b GetBit
addx.b d6,d6
bcc.b .getlit
move.b d6,(a1)+
.switch:
bsr.b GetKind
bcc.b .lit
.ref:
moveq.l #-1,d6
bsr.b GetBit
bcs.b .sameoffset
.newref:
moveq.l #3,d6
bsr.b GetNumber
moveq.l #2,d5
sub.l d7,d5
beq.b .end
.sameoffset:
moveq.l #4,d6
bsr.b GetNumber
.copyloop:
move.b (a1,d5.l),(a1)+
subq.l #1,d7
bne.b .copyloop
.afterref:
bsr.b GetKind
bcc.b .lit
bra.b .newref
.end:
move.l $4.w,a6
cmp.w #37,LIB_VERSION(a6)
blt.b not204
jmp CacheClearU(a6)
GetKind:
move.l a1,d4
moveq.l #1,d6
and.l d4,d6
lsl.w #8,d6
bra.b GetBit
GetNumber:
; D6 = Number context
; Out: Number in D7
lsl.w #8,d6
.numberloop:
addq.b #2,d6
bsr.b GetBit
bcs.b .numberloop
moveq.l #1,d7
subq.b #1,d6
.bitsloop:
bsr.b GetBit
addx.l d7,d7
subq.b #2,d6
bcc.b .bitsloop
not204: rts
; D6 = Bit context
; D1 = Input bit buffer
; D2 = Range value
; D3 = Interval size
; Out: Bit in C and X
readbit:
add.l d1,d1
bne.b nonewword
move.l -(a4),d1
addx.l d1,d1
nonewword:
addx.w d2,d2
add.w d3,d3
GetBit:
tst.w d3
bpl.b readbit
lea.l SINGLE_BIT_CONTEXTS*2(a2,d6.l),a5
add.l d6,a5
move.w (a5),d4
; D4 = One prob
lsr.w #ADJUST_SHIFT,d4
sub.w d4,(a5)
add.w (a5),d4
mulu.w d3,d4
swap.w d4
sub.w d4,d2
blo.b .one
.zero:
; oneprob = oneprob * (1 - adjust) = oneprob - oneprob * adjust
sub.w d4,d3
; 0 in C and X
rts
.one:
; onebrob = 1 - (1 - oneprob) * (1 - adjust) = oneprob - oneprob * adjust + adjust
add.w #$ffff>>ADJUST_SHIFT,(a5)
move.w d4,d3
add.w d4,d2
; 1 in C and X
rts
align 0,4
MiniHeader_End:
printv MiniHeader_End-MiniHeader
printt
printv ContextOffsetInstr+2-MiniHeader
Binary file not shown.
+1
View File
@@ -0,0 +1 @@
0x24, 0x13, 0xE5, 0x8A, 0x22, 0x42, 0x58, 0x89, 0x48, 0x51, 0x45, 0xFA, 0x20, 0xA8, 0x72, 0x00, 0x35, 0x3C, 0x80, 0x00, 0x50, 0x41, 0x6A, 0xF8, 0x28, 0x4A, 0x48, 0x41, 0x76, 0x01, 0x7C, 0x00, 0x52, 0x06, 0x61, 0x68, 0xDD, 0x06, 0x64, 0xFA, 0x12, 0xC6, 0x61, 0x34, 0x64, 0xF2, 0x7C, 0xFF, 0x61, 0x5A, 0x65, 0x0A, 0x7C, 0x03, 0x61, 0x32, 0x7A, 0x02, 0x9A, 0x87, 0x67, 0x12, 0x7C, 0x04, 0x61, 0x28, 0x12, 0xF1, 0x58, 0x00, 0x53, 0x87, 0x66, 0xF8, 0x61, 0x14, 0x64, 0xD2, 0x60, 0xE4, 0x2C, 0x78, 0x00, 0x04, 0x0C, 0x6E, 0x00, 0x25, 0x00, 0x14, 0x6D, 0x22, 0x4E, 0xEE, 0xFD, 0x84, 0x28, 0x09, 0x7C, 0x01, 0xCC, 0x84, 0xE1, 0x4E, 0x60, 0x22, 0xE1, 0x4E, 0x54, 0x06, 0x61, 0x1C, 0x65, 0xFA, 0x7E, 0x01, 0x53, 0x06, 0x61, 0x14, 0xDF, 0x87, 0x55, 0x06, 0x64, 0xF8, 0x4E, 0x75, 0xD2, 0x81, 0x66, 0x04, 0x22, 0x24, 0xD3, 0x81, 0xD5, 0x42, 0xD6, 0x43, 0x4A, 0x43, 0x6A, 0xF0, 0x4B, 0xF2, 0x68, 0x02, 0xDB, 0xC6, 0x38, 0x15, 0xE8, 0x4C, 0x99, 0x55, 0xD8, 0x55, 0xC8, 0xC3, 0x48, 0x44, 0x94, 0x44, 0x65, 0x04, 0x96, 0x44, 0x4E, 0x75, 0x06, 0x55, 0x0F, 0xFF, 0x36, 0x04, 0xD4, 0x44, 0x4E, 0x75,
+230
View File
@@ -0,0 +1,230 @@
; Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
TEXT = 0
; auto wb\OverlapHeader\OverlapHeader_End\
INIT_ONE_PROB = $8000
ADJUST_SHIFT = 4
SINGLE_BIT_CONTEXTS = 1
NUM_CONTEXTS = 1536
DUMMY_TEXT_LENGTH = 0
; Exec
LIB_VERSION = 20
OldOpenLibrary = -408
CloseLibrary = -414
CacheClearU = -636
; Dos
Write = -48
Output = -60
align 0,4
OverlapHeader:
move.l (a3),d2
lsl.l #2,d2
move.l d2,a3
move.l $4.w,a6
if TEXT
movem.l d0/a0,-(a7)
TextLengthInstr:
move.l #DUMMY_TEXT_LENGTH,d3
lea.l DosName(pc),a1
jsr OldOpenLibrary(a6)
move.l d0,a6
jsr Output(a6)
move.l d0,d1
beq.b noout
TextOffsetInstr:
lea.l OverlapHeader_End(pc),a0
move.l a0,d2
jsr Write(a6)
noout: move.l a6,a1
move.l $4.w,a6
jsr CloseLibrary(a6)
movem.l (a7)+,d0/a0
endc
; Init range decoder state
moveq.l #1,d1
ror.l #1,d1
moveq.l #1,d3
; Lowest bit of D2 = 0
move.l a3,a2
HunkLoop:
; Move packed data to end of hunk
move.l a2,a4
add.l -(a4),a4
lea.l 4(a2),a1
move.l (a1),d6
.move: move.l (a1,d6.l),-(a4)
subq.l #4,d6
bgt.b .move
; A4 = Start of packed data hunk
; A1 = Hunk Data Destination
moveq.l #NUM_CONTEXTS>>4,d6
lsl.l #4,d6
.init: move.w #INIT_ONE_PROB,-(a7)
subq.w #1,d6
bne.b .init
; moveq.l #0,d6
.lit:
addq.b #1,d6
.getlit:
bsr.b GetBit
addx.b d6,d6
bcc.b .getlit
move.b d6,(a1)+
.switch:
bsr.b GetKind
bcc.b .lit
.ref:
moveq.l #-1,d6
bsr.b GetBit
bcs.b .sameoffset
.newref:
moveq.l #3,d6
bsr.b GetNumber
moveq.l #2,d5
sub.l d7,d5
beq.b .hunkend
.sameoffset:
moveq.l #4,d6
bsr.b GetNumber
.copyloop:
move.b (a1,d5.l),(a1)+
subq.l #1,d7
bne.b .copyloop
.afterref:
bsr.b GetKind
bcc.b .lit
bra.b .newref
.hunkend:
; Relocs
move.l a3,d5
RelocHunk:
addq.l #4,d5
move.l a2,a1
.relocloop:
moveq.l #5,d6
bsr.b GetNumber
add.l d7,a1
lsr.l #2,d7
beq.b NextRelocHunk
add.l d5,(a1)
bra.b .relocloop
NextRelocHunk:
move.l d5,a1
move.l -(a1),d5
lsl.l #2,d5
bne.b RelocHunk
NextHunk:
lea.l NUM_CONTEXTS*2(a7),a7
move.l (a2),d4
lsl.l #2,d4
move.l d4,a2
bne.b HunkLoop
End:
cmp.w #37,LIB_VERSION(a6)
blt.b .not204
jsr CacheClearU(a6)
.not204:
jmp 4(a3)
if TEXT
DosName:
dc.b "dos.library",0
endc
GetKind:
move.l a1,d4
moveq.l #1,d6
and.l d4,d6
lsl.w #8,d6
GetBit: bra.b GetBitInner
GetNumber:
; D6 = Number context
; Out: Number in D7
lsl.w #8,d6
.numberloop:
addq.b #2,d6
bsr.b GetBitInner
bcs.b .numberloop
moveq.l #1,d7
subq.b #1,d6
.bitsloop:
bsr.b GetBitInner
addx.l d7,d7
subq.b #2,d6
bcc.b .bitsloop
rts
; D6 = Bit context
; D1 = Input bit buffer
; D2 = Range value
; D3 = Interval size
; Out: Bit in C and X
readbit:
add.l d1,d1
bne.b nonewword
move.l (a4)+,d1
addx.l d1,d1
nonewword:
addx.w d2,d2
add.w d3,d3
GetBitInner:
tst.w d3
bpl.b readbit
lea.l 4+SINGLE_BIT_CONTEXTS*2(a7,d6.l),a5
add.l d6,a5
move.w (a5),d4
; D4 = One prob
lsr.w #ADJUST_SHIFT,d4
sub.w d4,(a5)
add.w (a5),d4
mulu.w d3,d4
swap.w d4
sub.w d4,d2
blo.b .one
.zero:
; oneprob = oneprob * (1 - adjust) = oneprob - oneprob * adjust
sub.w d4,d3
; 0 in C and X
rts
.one:
; onebrob = 1 - (1 - oneprob) * (1 - adjust) = oneprob - oneprob * adjust + adjust
add.w #$ffff>>ADJUST_SHIFT,(a5)
move.w d4,d3
add.w d4,d2
; 1 in C and X
rts
align 0,4
OverlapHeader_End:
printv OverlapHeader_End-OverlapHeader
if TEXT
printt
printv TextLengthInstr+2-OverlapHeader
printv TextOffsetInstr+2-OverlapHeader
endc
Binary file not shown.
+1
View File
@@ -0,0 +1 @@
0x24, 0x13, 0xE5, 0x8A, 0x26, 0x42, 0x2C, 0x78, 0x00, 0x04, 0x72, 0x01, 0xE2, 0x99, 0x76, 0x01, 0x24, 0x4B, 0x28, 0x4A, 0xD9, 0xE4, 0x43, 0xEA, 0x00, 0x04, 0x2C, 0x11, 0x29, 0x31, 0x68, 0x00, 0x59, 0x86, 0x6E, 0xF8, 0x7C, 0x60, 0xE9, 0x8E, 0x3F, 0x3C, 0x80, 0x00, 0x53, 0x46, 0x66, 0xF8, 0x52, 0x06, 0x61, 0x6C, 0xDD, 0x06, 0x64, 0xFA, 0x12, 0xC6, 0x61, 0x5C, 0x64, 0xF2, 0x7C, 0xFF, 0x61, 0x5E, 0x65, 0x0A, 0x7C, 0x03, 0x61, 0x5A, 0x7A, 0x02, 0x9A, 0x87, 0x67, 0x12, 0x7C, 0x04, 0x61, 0x50, 0x12, 0xF1, 0x58, 0x00, 0x53, 0x87, 0x66, 0xF8, 0x61, 0x3C, 0x64, 0xD2, 0x60, 0xE4, 0x2A, 0x0B, 0x58, 0x85, 0x22, 0x4A, 0x7C, 0x05, 0x61, 0x38, 0xD3, 0xC7, 0xE4, 0x8F, 0x67, 0x04, 0xDB, 0x91, 0x60, 0xF2, 0x22, 0x45, 0x2A, 0x21, 0xE5, 0x8D, 0x66, 0xE6, 0x4F, 0xEF, 0x0C, 0x00, 0x28, 0x12, 0xE5, 0x8C, 0x24, 0x44, 0x66, 0x8A, 0x0C, 0x6E, 0x00, 0x25, 0x00, 0x14, 0x6D, 0x04, 0x4E, 0xAE, 0xFD, 0x84, 0x4E, 0xEB, 0x00, 0x04, 0x28, 0x09, 0x7C, 0x01, 0xCC, 0x84, 0xE1, 0x4E, 0x60, 0x22, 0xE1, 0x4E, 0x54, 0x06, 0x61, 0x1C, 0x65, 0xFA, 0x7E, 0x01, 0x53, 0x06, 0x61, 0x14, 0xDF, 0x87, 0x55, 0x06, 0x64, 0xF8, 0x4E, 0x75, 0xD2, 0x81, 0x66, 0x04, 0x22, 0x1C, 0xD3, 0x81, 0xD5, 0x42, 0xD6, 0x43, 0x4A, 0x43, 0x6A, 0xF0, 0x4B, 0xF7, 0x68, 0x06, 0xDB, 0xC6, 0x38, 0x15, 0xE8, 0x4C, 0x99, 0x55, 0xD8, 0x55, 0xC8, 0xC3, 0x48, 0x44, 0x94, 0x44, 0x65, 0x04, 0x96, 0x44, 0x4E, 0x75, 0x06, 0x55, 0x0F, 0xFF, 0x36, 0x04, 0xD4, 0x44, 0x4E, 0x75,
Binary file not shown.
+1
View File
@@ -0,0 +1 @@
0x24, 0x13, 0xE5, 0x8A, 0x26, 0x42, 0x2C, 0x78, 0x00, 0x04, 0x48, 0xE7, 0x80, 0x80, 0x26, 0x3C, 0x00, 0x00, 0x00, 0x00, 0x43, 0xFA, 0x00, 0xB6, 0x4E, 0xAE, 0xFE, 0x68, 0x2C, 0x40, 0x4E, 0xAE, 0xFF, 0xC4, 0x22, 0x00, 0x67, 0x0A, 0x41, 0xFA, 0x01, 0x04, 0x24, 0x08, 0x4E, 0xAE, 0xFF, 0xD0, 0x22, 0x4E, 0x2C, 0x78, 0x00, 0x04, 0x4E, 0xAE, 0xFE, 0x62, 0x4C, 0xDF, 0x01, 0x01, 0x72, 0x01, 0xE2, 0x99, 0x76, 0x01, 0x24, 0x4B, 0x28, 0x4A, 0xD9, 0xE4, 0x43, 0xEA, 0x00, 0x04, 0x2C, 0x11, 0x29, 0x31, 0x68, 0x00, 0x59, 0x86, 0x6E, 0xF8, 0x7C, 0x60, 0xE9, 0x8E, 0x3F, 0x3C, 0x80, 0x00, 0x53, 0x46, 0x66, 0xF8, 0x52, 0x06, 0x61, 0x78, 0xDD, 0x06, 0x64, 0xFA, 0x12, 0xC6, 0x61, 0x68, 0x64, 0xF2, 0x7C, 0xFF, 0x61, 0x6A, 0x65, 0x0A, 0x7C, 0x03, 0x61, 0x66, 0x7A, 0x02, 0x9A, 0x87, 0x67, 0x12, 0x7C, 0x04, 0x61, 0x5C, 0x12, 0xF1, 0x58, 0x00, 0x53, 0x87, 0x66, 0xF8, 0x61, 0x48, 0x64, 0xD2, 0x60, 0xE4, 0x2A, 0x0B, 0x58, 0x85, 0x22, 0x4A, 0x7C, 0x05, 0x61, 0x44, 0xD3, 0xC7, 0xE4, 0x8F, 0x67, 0x04, 0xDB, 0x91, 0x60, 0xF2, 0x22, 0x45, 0x2A, 0x21, 0xE5, 0x8D, 0x66, 0xE6, 0x4F, 0xEF, 0x0C, 0x00, 0x28, 0x12, 0xE5, 0x8C, 0x24, 0x44, 0x66, 0x8A, 0x0C, 0x6E, 0x00, 0x25, 0x00, 0x14, 0x6D, 0x04, 0x4E, 0xAE, 0xFD, 0x84, 0x4E, 0xEB, 0x00, 0x04, 0x64, 0x6F, 0x73, 0x2E, 0x6C, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x00, 0x28, 0x09, 0x7C, 0x01, 0xCC, 0x84, 0xE1, 0x4E, 0x60, 0x22, 0xE1, 0x4E, 0x54, 0x06, 0x61, 0x1C, 0x65, 0xFA, 0x7E, 0x01, 0x53, 0x06, 0x61, 0x14, 0xDF, 0x87, 0x55, 0x06, 0x64, 0xF8, 0x4E, 0x75, 0xD2, 0x81, 0x66, 0x04, 0x22, 0x1C, 0xD3, 0x81, 0xD5, 0x42, 0xD6, 0x43, 0x4A, 0x43, 0x6A, 0xF0, 0x4B, 0xF7, 0x68, 0x06, 0xDB, 0xC6, 0x38, 0x15, 0xE8, 0x4C, 0x99, 0x55, 0xD8, 0x55, 0xC8, 0xC3, 0x48, 0x44, 0x94, 0x44, 0x65, 0x04, 0x96, 0x44, 0x4E, 0x75, 0x06, 0x55, 0x0F, 0xFF, 0x36, 0x04, 0xD4, 0x44, 0x4E, 0x75,
+133
View File
@@ -0,0 +1,133 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
Pack a data block in multiple iterations, reporting progress along the way.
*/
#pragma once
#include "RangeCoder.h"
#include "MatchFinder.h"
#include "CountingCoder.h"
#include "SizeMeasuringCoder.h"
#include "LZEncoder.h"
#include "LZParser.h"
struct PackParams {
int iterations;
int length_margin;
int skip_length;
int match_patience;
int max_same_length;
};
class PackProgress : public LZProgress {
int size;
int steps;
int next_step_threshold;
int textlength;
void print() {
textlength = printf("[%d.%d%%]", steps / 10, steps % 10);
fflush(stdout);
}
void rewind() {
printf("\033[%dD", textlength);
}
public:
virtual void begin(int size) {
this->size = size;
steps = 0;
next_step_threshold = size / 1000;
print();
}
virtual void update(int pos) {
if (pos < next_step_threshold) return;
while (pos >= next_step_threshold) {
steps += 1;
next_step_threshold = (long long) size * (steps + 1) / 1000;
}
rewind();
print();
}
virtual void end() {
rewind();
printf("\033[K");
fflush(stdout);
}
};
class NoProgress : public LZProgress {
public:
virtual void begin(int size) {
fflush(stdout);
}
virtual void update(int pos) {
}
virtual void end() {
}
};
void packData(unsigned char *data, int data_length, int zero_padding, PackParams *params, Coder *result_coder, RefEdgeFactory *edge_factory, bool show_progress) {
MatchFinder finder(data, data_length, 2, params->match_patience, params->max_same_length);
LZParser parser(data, data_length, zero_padding, finder, params->length_margin, params->skip_length, edge_factory);
int real_size = 0;
int best_size = 999999999;
int best_result = 0;
vector<LZParseResult> results(2);
CountingCoder *counting_coder = new CountingCoder(LZEncoder::NUM_CONTEXTS);
LZProgress *progress;
if (show_progress) {
progress = new PackProgress();
} else {
progress = new NoProgress();
}
printf("%8d", data_length);
for (int i = 0 ; i < params->iterations ; i++) {
printf(" ");
// Parse data into LZ symbols
LZParseResult& result = results[1 - best_result];
Coder *measurer = new SizeMeasuringCoder(counting_coder);
measurer->setNumberContexts(LZEncoder::NUMBER_CONTEXT_OFFSET, LZEncoder::NUM_NUMBER_CONTEXTS, data_length);
finder.reset();
result = parser.parse(LZEncoder(measurer), progress);
// Encode result using adaptive range coding
vector<unsigned> dummy_result;
RangeCoder *range_coder = new RangeCoder(LZEncoder::NUM_CONTEXTS, dummy_result);
real_size = result.encode(LZEncoder(range_coder));
range_coder->finish();
delete range_coder;
// Choose if best
if (real_size < best_size) {
best_result = 1 - best_result;
best_size = real_size;
}
// Print size
printf("%14.3f", real_size / (double) (8 << Coder::BIT_PRECISION));
// Count symbol frequencies
CountingCoder *new_counting_coder = new CountingCoder(LZEncoder::NUM_CONTEXTS);
result.encode(LZEncoder(counting_coder));
// New size measurer based on frequencies
CountingCoder *old_counting_coder = counting_coder;
counting_coder = new CountingCoder(old_counting_coder, new_counting_coder);
delete old_counting_coder;
delete new_counting_coder;
}
delete progress;
delete counting_coder;
results[best_result].encode(LZEncoder(result_coder));
}
+62
View File
@@ -0,0 +1,62 @@
Shrinkler executable file compressor for Amiga by Blueberry
Designed for maximum compression of Amiga 64k and 4k intros, and
everything in between.
Executables for different platforms are available in their respective
subdirectories. The output executables are compatible with all Amiga CPUs
and kickstarts.
Run with no arguments for a list of options. For the options controlling
compression efficiency, higher values generally result in better
compression, at the cost of higher time and/or memory requirements.
History:
2015-01-18: Version 4.4. Optimizations galore:
New match finder based on a suffix array.
New reference edge map based on a cuckoo hash table.
Pre-compute number encoding sizes for faster estimation.
Recycle references edges to save alloc/dealloc overhead.
Updated defaults to take advantage of speed increase.
Data file compression mode with decompression source.
Fixed broken progress output for big files.
Do not crash if text file could not be opened.
2015-01-05: Version 4.3. Minor fixes:
Usage information adjusted to fit within 77 columns.
References discarded metric computed properly.
First progress step is at 0.1% rather than 1.0%.
Option to omit progress output (for non-ANSI consoles).
Source changes for easier compilation with MSVC.
2014-12-16: Version 4.2. For memory-efficient decrunching:
Option to overlap compressed and decompressed data.
Print memory overhead during and after decrunching.
Verifier accepts partially filled hunks.
2014-02-08: Version 4.1. Bug fixes and new features:
Fixed some bugs in the range coder.
Fixed handling of very small first hunk.
Added internal verifier to check correctness of output.
Print helpful text when encountering an internal error.
Better error message when running out of memory.
Set output file to be executable.
New options to print text from an argument or file.
New option to flash a hardware register during decrunching.
2014-01-05: Version 4.0. First public release with new name.
1999 - 2012: Various public and internal versions.
Source code available from https://bitbucket.org/askeksa/shrinkler
For questions and comments, visit the ADA forum at
http://ada.untergrund.net/?p=boardthread&id=264
or write to blueberry at loonies dot dk.
+135
View File
@@ -0,0 +1,135 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
An entropy coder based on range coding.
*/
#pragma once
#include <cassert>
#include <cmath>
#include <algorithm>
#include <vector>
using std::fill;
using std::vector;
#include "Coder.h"
#ifndef ADJUST_SHIFT
#define ADJUST_SHIFT 4
#endif
class RangeCoder : public Coder {
vector<unsigned short> contexts;
vector<unsigned>& out;
int dest_bit;
unsigned intervalsize;
unsigned intervalmin;
static int sizetable[128];
static bool sizetable_init;
static bool init_sizetable() {
for (int i = 0 ; i < 128 ; i++) {
sizetable[i] = (int) floor(0.5 + (8.0 - log((double) (128 + i)) / log(2.0)) * (1 << BIT_PRECISION));
}
return true;
}
void addBit() {
int pos = dest_bit;
int longpos;
int bitmask;
do {
pos--;
if (pos < 0) return;
longpos = pos >> 5;
bitmask = 0x80000000 >> (pos & 31);
while (longpos >= out.size()) {
out.push_back(0);
}
out[longpos] ^= bitmask;
} while ((out[longpos] & bitmask) == 0);
}
public:
RangeCoder(int n_contexts, vector<unsigned>& out) : out(out) {
contexts.resize(n_contexts, 0x8000);
dest_bit = -1;
intervalsize = 0x8000;
intervalmin = 0;
out.clear();
}
virtual int code(int context_index, int bit) {
assert(context_index < contexts.size());
assert(bit == 0 || bit == 1);
int size_before = (dest_bit << BIT_PRECISION) + sizetable[(intervalsize - 0x8000) >> 8];
unsigned prob = contexts[context_index];
unsigned threshold = (intervalsize * prob) >> 16;
unsigned new_prob;
if (!bit) {
// Zero
intervalmin += threshold;
if (intervalmin & 0x10000) {
addBit();
}
intervalsize = intervalsize - threshold;
new_prob = prob - (prob >> ADJUST_SHIFT);
} else {
// One
intervalsize = threshold;
new_prob = prob + (0xffff >> ADJUST_SHIFT) - (prob >> ADJUST_SHIFT);
}
assert(new_prob > 0);
assert(new_prob < 0x10000);
contexts[context_index] = new_prob;
while (intervalsize < 0x8000) {
dest_bit++;
intervalsize <<= 1;
intervalmin <<= 1;
if (intervalmin & 0x10000) {
addBit();
}
}
intervalmin &= 0xffff;
int size_after = (dest_bit << BIT_PRECISION) + sizetable[(intervalsize - 0x8000) >> 8];
return size_after - size_before;
}
void reset() {
fill(contexts.begin(), contexts.end(), 0x8000);
}
void finish() {
int intervalmax = intervalmin + intervalsize;
int final_min = 0;
int final_size = 0x10000;
while (final_min < intervalmin || final_min + final_size >= intervalmax) {
if (final_min + final_size < intervalmax) {
addBit();
final_min += final_size;
}
dest_bit++;
final_size >>= 1;
}
while ((dest_bit - 1) >> 5 >= out.size()) {
out.push_back(0);
}
}
int sizeInBits() {
return dest_bit + 1;
}
};
int RangeCoder::sizetable[128];
bool RangeCoder::sizetable_init = init_sizetable();
+103
View File
@@ -0,0 +1,103 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
A decoder for the range coder.
*/
#pragma once
#include <cmath>
#include <algorithm>
#include <vector>
using std::fill;
using std::vector;
#include "Decoder.h"
#include "assert.h"
#ifndef ADJUST_SHIFT
#define ADJUST_SHIFT 4
#endif
class CompressedDataReadListener {
public:
virtual void read(int index) = 0;
virtual ~CompressedDataReadListener() {}
};
class RangeDecoder : public Decoder {
vector<unsigned short> contexts;
vector<unsigned>& data;
CompressedDataReadListener* listener;
int bit_index;
unsigned intervalsize;
unsigned intervalvalue;
unsigned uncertainty;
int getBit() {
int long_index = bit_index >> 5;
int bit_in_long = (~bit_index) & 31;
if (bit_in_long == 31) {
if (listener) listener->read(long_index);
}
if (bit_index++ >= data.size() * 32) {
uncertainty <<= 1;
return 0;
}
int bit = (data[long_index] >> bit_in_long) & 1;
return bit;
}
public:
RangeDecoder(int n_contexts, vector<unsigned>& data) : data(data) {
contexts.resize(n_contexts, 0x8000);
bit_index = 0;
intervalsize = 1;
intervalvalue = 0;
uncertainty = 1;
listener = NULL;
}
virtual int decode(int context_index) {
assert(context_index < contexts.size());
unsigned prob = contexts[context_index];
while (intervalsize < 0x8000) {
intervalsize <<= 1;
intervalvalue = (intervalvalue << 1) | getBit();
}
int bit;
unsigned new_prob;
unsigned threshold = (intervalsize * prob) >> 16;
if (intervalvalue >= threshold) {
// Zero
bit = 0;
intervalvalue -= threshold;
intervalsize -= threshold;
new_prob = prob - (prob >> ADJUST_SHIFT);
} else {
// One
assert(intervalvalue + uncertainty <= threshold);
bit = 1;
intervalsize = threshold;
new_prob = prob + (0xffff >> ADJUST_SHIFT) - (prob >> ADJUST_SHIFT);
}
assert(new_prob > 0);
assert(new_prob < 0x10000);
contexts[context_index] = new_prob;
return bit;
}
void reset() {
fill(contexts.begin(), contexts.end(), 0x8000);
}
void setListener(CompressedDataReadListener* listener) {
this->listener = listener;
}
};
+361
View File
@@ -0,0 +1,361 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
Main file for the cruncher.
*/
//#define SHRINKLER_TITLE ("Shrinkler executable file compressor by Blueberry - version 4.4 (2015-01-18)\n\n")
#ifndef SHRINKLER_TITLE
#define SHRINKLER_TITLE ("Shrinkler executable file compressor by Blueberry - development version (built " __DATE__ " " __TIME__ ")\n\n")
#endif
#include <cstdio>
#include <cstdlib>
#include <string>
#include <sys/stat.h>
using std::string;
#include "HunkFile.h"
#include "DataFile.h"
void usage() {
printf("Usage: Shrinkler <options> <input executable> <output executable>\n");
printf("\n");
printf("Available options are (default values in parentheses):\n");
printf(" -d, --data Treat input as raw data, rather than executable\n");
printf(" -h, --hunkmerge Merge hunks of the same memory type\n");
printf(" -o, --overlap Overlap compressed and decompressed data to save memory\n");
printf(" -m, --mini Use a smaller, but more restricted decrunch header\n");
printf(" -i, --iterations Number of iterations for the compression (2)\n");
printf(" -l, --length-margin Number of shorter matches considered for each match (2)\n");
printf(" -a, --same-length Number of matches of the same length to consider (20)\n");
printf(" -e, --effort Perseverance in finding multiple matches (200)\n");
printf(" -s, --skip-length Minimum match length to accept greedily (2000)\n");
printf(" -r, --references Number of reference edges to keep in memory (100000)\n");
printf(" -t, --text Print a text, followed by a newline, before decrunching\n");
printf(" -T, --textfile Print the contents of the given file before decrunching\n");
printf(" -f, --flash Poke into a register (e.g. DFF180) during decrunching\n");
printf(" -p, --no-progress Do not print progress info: no ANSI codes in output\n");
printf("\n");
exit(0);
}
class Parameter {
public:
bool seen;
virtual ~Parameter() {}
protected:
void parse(const char *form1, const char *form2, const char *arg_kind, int argc, const char *argv[], vector<bool>& consumed) {
seen = false;
for (int i = 1 ; i < argc ; i++) {
if (strcmp(argv[i], form1) == 0 || strcmp(argv[i], form2) == 0) {
if (seen) {
printf("Error: %s specified multiple times.\n\n", argv[i]);
usage();
}
consumed[i] = true;
if (arg_kind) {
if (i+1 < argc && !consumed[i+1] && argv[i+1][0] != '-') {
seen = parseArg(argv[i], argv[i+1]);
}
if (!seen) {
printf("Error: %s requires a %s argument.\n\n", argv[i], arg_kind);
usage();
}
consumed[i+1] = true;
i = i+1;
} else {
seen = true;
}
}
}
}
virtual bool parseArg(const char *param, const char *arg) = 0;
};
class IntParameter : public Parameter {
int min_value;
int max_value;
public:
int value;
IntParameter(const char *form1, const char *form2, int min_value, int max_value, int default_value,
int argc, const char *argv[], vector<bool>& consumed)
: min_value(min_value), max_value(max_value), value(default_value)
{
parse(form1, form2, "numeric", argc, argv, consumed);
}
protected:
virtual bool parseArg(const char *param, const char *arg) {
char *endptr;
value = strtol(arg, &endptr, 10);
if (endptr == &arg[strlen(arg)]) {
if (value < min_value || value > max_value) {
printf("Error: Argument of %s must be between %d and %d.\n\n", param, min_value, max_value);
usage();
}
return true;
}
return false;
}
};
class HexParameter : public Parameter {
public:
unsigned value;
HexParameter(const char *form1, const char *form2, int default_value,
int argc, const char *argv[], vector<bool>& consumed)
: value(default_value)
{
parse(form1, form2, "hexadecimal", argc, argv, consumed);
}
protected:
virtual bool parseArg(const char *param, const char *arg) {
char *endptr;
value = strtol(arg, &endptr, 16);
if (endptr == &arg[strlen(arg)]) {
return true;
}
return false;
}
};
class StringParameter : public Parameter {
public:
const char *value;
StringParameter(const char *form1, const char *form2, int argc, const char *argv[], vector<bool>& consumed)
: value(NULL)
{
parse(form1, form2, "string", argc, argv, consumed);
}
protected:
virtual bool parseArg(const char *param, const char *arg) {
value = arg;
return true;
}
};
class FlagParameter : public Parameter {
public:
FlagParameter(const char *form1, const char *form2, int argc, const char *argv[], vector<bool>& consumed)
{
parse(form1, form2, NULL, argc, argv, consumed);
}
protected:
virtual bool parseArg(const char *param, const char *arg) {
// Not used
return true;
}
};
int main2(int argc, const char *argv[]) {
printf(SHRINKLER_TITLE);
vector<bool> consumed(argc);
FlagParameter data ("-d", "--data", argc, argv, consumed);
FlagParameter hunkmerge ("-h", "--hunkmerge", argc, argv, consumed);
FlagParameter overlap ("-o", "--overlap", argc, argv, consumed);
FlagParameter mini ("-m", "--mini", argc, argv, consumed);
IntParameter iterations ("-i", "--iterations", 1, 9, 2, argc, argv, consumed);
IntParameter length_margin ("-l", "--length-margin", 0, 100, 2, argc, argv, consumed);
IntParameter same_length ("-a", "--same-length", 1, 100000, 20, argc, argv, consumed);
IntParameter effort ("-e", "--effort", 0, 100000, 200, argc, argv, consumed);
IntParameter skip_length ("-s", "--skip-length", 2, 100000, 2000, argc, argv, consumed);
IntParameter references ("-r", "--references", 1000, 10000000, 100000, argc, argv, consumed);
StringParameter text ("-t", "--text", argc, argv, consumed);
StringParameter textfile ("-T", "--textfile", argc, argv, consumed);
HexParameter flash ("-f", "--flash", 0, argc, argv, consumed);
FlagParameter no_progress ("-p", "--no-progress", argc, argv, consumed);
vector<const char*> files;
for (int i = 1 ; i < argc ; i++) {
if (!consumed[i]) {
if (argv[i][0] == '-') {
printf("Error: Unknown option %s\n\n", argv[i]);
usage();
}
files.push_back(argv[i]);
}
}
if (data.seen && (hunkmerge.seen || overlap.seen || mini.seen || text.seen || textfile.seen || flash.seen)) {
printf("Error: The data option cannot be used together with any of the\n");
printf("hunkmerge, overlap, mini, text, textfile or flash options.\n\n");
usage();
}
if (overlap.seen && mini.seen) {
printf("Error: The overlap and mini options cannot be used together.\n\n");
usage();
}
if (text.seen && textfile.seen) {
printf("Error: The text and textfile options cannot both be specified.\n\n");
usage();
}
if (mini.seen && (text.seen || textfile.seen)) {
printf("Error: The text and textfile options cannot be used in mini mode.\n\n");
usage();
}
if (files.size() == 0) {
printf("Error: No input file specified.\n\n");
usage();
}
if (files.size() == 1) {
printf("Error: No output file specified.\n\n");
usage();
}
if (files.size() > 2) {
printf("Error: Too many files specified.\n\n");
usage();
}
const char *infile = files[0];
const char *outfile = files[1];
PackParams params;
params.iterations = iterations.value;
params.length_margin = length_margin.value;
params.skip_length = skip_length.value;
params.match_patience = effort.value;
params.max_same_length = same_length.value;
string *decrunch_text_ptr = NULL;
string decrunch_text;
if (text.seen) {
decrunch_text = text.value;
decrunch_text.push_back('\n');
decrunch_text_ptr = &decrunch_text;
} else if (textfile.seen) {
FILE *decrunch_text_file = fopen(textfile.value, "r");
if (!decrunch_text_file) {
printf("Error: Could not open text file %s\n", textfile.value);
exit(1);
}
char c;
while ((c = fgetc(decrunch_text_file)) != EOF) {
decrunch_text.push_back(c);
}
fclose(decrunch_text_file);
decrunch_text_ptr = &decrunch_text;
}
if (data.seen) {
// Data file compression
printf("Loading file %s...\n\n", infile);
DataFile *orig = new DataFile;
orig->load(infile);
printf("Crunching...\n\n");
RefEdgeFactory edge_factory(references.value);
DataFile *crunched = orig->crunch(&params, &edge_factory, !no_progress.seen);
delete orig;
printf("References considered:%8d\n", edge_factory.max_edge_count);
printf("References discarded:%9d\n\n", edge_factory.max_cleaned_edges);
printf("Saving file %s...\n\n", outfile);
crunched->save(outfile);
printf("Final file size: %d\n\n", crunched->size());
delete crunched;
if (edge_factory.max_edge_count > references.value) {
printf("Note: compression may benefit from a larger reference buffer (-r option).\n\n");
}
return 0;
}
// Executable file compression
printf("Loading file %s...\n\n", infile);
HunkFile *orig = new HunkFile;
orig->load(infile);
if (!orig->analyze()) {
printf("\nError while analyzing input file!\n\n");
delete orig;
exit(1);
}
if (hunkmerge.seen) {
printf("Merging hunks...\n\n");
HunkFile *merged = orig->merge_hunks(orig->merged_hunklist());
delete orig;
if (!merged->analyze()) {
printf("\nError while analyzing merged file!\n\n");
delete merged;
internal_error();
}
orig = merged;
}
if (mini.seen && !orig->valid_mini()) {
printf("Input executable not suitable for mini crunching.\n"
"Must contain only one non-empty hunk and no relocations,\n"
"and the final file size must be less than 24k.\n\n");
delete orig;
exit(1);
}
int orig_mem = orig->memory_usage(true);
printf("Crunching...\n\n");
RefEdgeFactory edge_factory(references.value);
HunkFile *crunched = orig->crunch(&params, overlap.seen, mini.seen, decrunch_text_ptr, flash.value, &edge_factory, !no_progress.seen);
delete orig;
printf("References considered:%8d\n", edge_factory.max_edge_count);
printf("References discarded:%9d\n\n", edge_factory.max_cleaned_edges);
if (!crunched->analyze()) {
printf("\nError while analyzing crunched file!\n\n");
delete crunched;
internal_error();
}
int crunched_mem_during = crunched->memory_usage(true);
int crunched_mem_after = crunched->memory_usage(mini.seen || overlap.seen);
printf("Memory overhead during decrunching: %9d\n", crunched_mem_during - orig_mem);
printf("Memory overhead after decrunching: %9d\n\n", crunched_mem_after - orig_mem);
printf("Saving file %s...\n\n", outfile);
crunched->save(outfile);
#ifdef S_IRWXU // Is the POSIX file permission API available?
chmod(outfile, 0755); // Mark file executable
#endif
printf("Final file size: %d\n\n", crunched->size());
delete crunched;
if (edge_factory.max_edge_count > references.value) {
printf("Note: compression may benefit from a larger reference buffer (-r option).\n\n");
}
return 0;
}
int main(int argc, const char *argv[]) {
try {
return main2(argc, argv);
} catch (std::bad_alloc& e) {
fflush(stdout);
fprintf(stderr,
"\n\nShrinkler ran out of memory.\n\n"
"Some things you can try:\n"
" - Free up some memory\n"
" - Run it on a machine with more memory\n"
" - Reduce the size of the reference buffer (-r option)\n"
" - Split up your biggest hunk into smaller ones\n\n");
fflush(stderr);
return 1;
}
}
+172
View File
@@ -0,0 +1,172 @@
; Copyright 1999-2015 Aske Simon Christensen.
;
; The code herein is free to use, in whole or in part,
; modified or as is, for any legal purpose.
;
; No warranties of any kind are given as to its behavior
; or suitability.
INIT_ONE_PROB = $8000
ADJUST_SHIFT = 4
SINGLE_BIT_CONTEXTS = 1
NUM_CONTEXTS = 1536
; Decompress Shrinkler-compressed data produced with the --data option.
;
; A0 = Compressed data
; A1 = Decompressed data destination
; A2 = Progress callback, can be zero if no callback is desired.
; Callback will be called continuously with
; D0 = Number of bytes decompressed so far
; A0 = Callback argument
; A3 = Callback argument
;
; Uses 3 kilobytes of space on the stack.
; Preserves D2-D7/A2-A6 and assumes callback does the same.
;
; Decompression code may read one longword beyond compressed data.
; The contents of this longword does not matter.
ShrinklerDecompress:
movem.l d2-d7/a4-a6,-(a7)
move.l a0,a4
move.l a1,a5
move.l a1,a6
; Init range decoder state
moveq.l #0,d2
moveq.l #1,d3
moveq.l #1,d4
ror.l #1,d4
; Init probabilities
move.l #NUM_CONTEXTS,d6
.init: move.w #INIT_ONE_PROB,-(a7)
subq.w #1,d6
bne.b .init
; D6 = 0
.lit:
; Literal
addq.b #1,d6
.getlit:
bsr.b GetBit
addx.b d6,d6
bcc.b .getlit
move.b d6,(a5)+
bsr.b ReportProgress
.switch:
; After literal
bsr.b GetKind
bcc.b .lit
; Reference
moveq.l #-1,d6
bsr.b GetBit
bcc.b .readoffset
.readlength:
moveq.l #4,d6
bsr.b GetNumber
.copyloop:
move.b (a5,d5.l),(a5)+
subq.l #1,d7
bne.b .copyloop
bsr.b ReportProgress
; After reference
bsr.b GetKind
bcc.b .lit
.readoffset:
moveq.l #3,d6
bsr.b GetNumber
moveq.l #2,d5
sub.l d7,d5
bne.b .readlength
lea.l NUM_CONTEXTS*2(a7),a7
movem.l (a7)+,d2-d7/a4-a6
rts
ReportProgress:
move.l a2,d0
beq.b .nocallback
move.l a5,d0
sub.l a6,d0
move.l a3,a0
jsr (a2)
.nocallback:
rts
GetKind:
; Use parity as context
move.l a5,d1
moveq.l #1,d6
and.l d1,d6
lsl.w #8,d6
bra.b GetBit
GetNumber:
; D6 = Number context
; Out: Number in D7
lsl.w #8,d6
.numberloop:
addq.b #2,d6
bsr.b GetBit
bcs.b .numberloop
moveq.l #1,d7
subq.b #1,d6
.bitsloop:
bsr.b GetBit
addx.l d7,d7
subq.b #2,d6
bcc.b .bitsloop
rts
; D6 = Bit context
; D2 = Range value
; D3 = Interval size
; D4 = Input bit buffer
; Out: Bit in C and X
readbit:
add.l d4,d4
bne.b nonewword
move.l (a4)+,d4
addx.l d4,d4
nonewword:
addx.w d2,d2
add.w d3,d3
GetBit:
tst.w d3
bpl.b readbit
lea.l 4+SINGLE_BIT_CONTEXTS*2(a7,d6.l),a1
add.l d6,a1
move.w (a1),d1
; D1 = One prob
lsr.w #ADJUST_SHIFT,d1
sub.w d1,(a1)
add.w (a1),d1
mulu.w d3,d1
swap.w d1
sub.w d1,d2
blo.b .one
.zero:
; oneprob = oneprob * (1 - adjust) = oneprob - oneprob * adjust
sub.w d1,d3
; 0 in C and X
rts
.one:
; onebrob = 1 - (1 - oneprob) * (1 - adjust) = oneprob - oneprob * adjust + adjust
add.w #$ffff>>ADJUST_SHIFT,(a1)
move.w d1,d3
add.w d1,d2
; 1 in C and X
rts
+60
View File
@@ -0,0 +1,60 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
A dummy entropy coder which estimates the size of coded symbols based on
counts from a CountingCoder.
*/
#pragma once
#include <vector>
using std::vector;
#include "CountingCoder.h"
struct ContextSizes {
unsigned short sizes[2];
};
class SizeMeasuringCoder : public Coder {
static const int MIN_SIZE = 2;
static const int MAX_SIZE = 12 << BIT_PRECISION;
vector<ContextSizes> context_sizes;
int sizeForCount(int count, int total) {
int size = (int) floor(0.5 + log(total / (double) count) / log(2.0) * (1 << BIT_PRECISION));
if (size < MIN_SIZE) size = MIN_SIZE;
if (size > MAX_SIZE) size = MAX_SIZE;
return size;
}
public:
SizeMeasuringCoder(int n_contexts) {
struct ContextSizes default_sizes = { { 1 << BIT_PRECISION, 1 << BIT_PRECISION } };
context_sizes.resize(n_contexts, default_sizes);
setCacheable(true);
}
SizeMeasuringCoder(CountingCoder *counting_coder) {
context_sizes.resize(counting_coder->context_counts.size());
for (int i = 0 ; i < counting_coder->context_counts.size() ; i++) {
struct ContextSizes s;
struct ContextCounts c = counting_coder->context_counts[i];
int count0 = 1 + c.counts[0];
int count1 = 1 + c.counts[1];
int sum = count0 + count1;
s.sizes[0] = sizeForCount(count0, sum);
s.sizes[1] = sizeForCount(count1, sum);
context_sizes[i] = s;
}
setCacheable(true);
}
virtual int code(int context_index, int bit) {
return context_sizes[context_index].sizes[bit];
}
};
+37
View File
@@ -0,0 +1,37 @@
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
/*
An assert function which contains a breakpoint, for ease of debugging.
*/
#pragma once
void internal_error() {
fflush(stdout);
fprintf(stderr,
"\n\nShrinkler has encountered an internal error.\n"
"Please send a bug report to blueberry@loonies.dk,\n"
"providing the file you tried to compress.\n"
"\n"
"Thanks, and apologies for the inconvenience.\n\n");
fflush(stderr);
exit(1);
}
#ifndef NDEBUG
#include <stdio.h>
static void _assert_func(const char *file, int line, const char *exp) {
fflush(stdout);
fprintf(stderr, "\n\nassertion \"%s\" failed: file \"%s\", line %d\n", exp, file, line);
fflush(stderr);
#ifdef DEBUG
__asm volatile ("int3;");
#endif
internal_error();
}
#undef assert
#define assert(__e) ((__e) ? (void)0 : _assert_func (__FILE__, __LINE__, #__e))
#endif
+95
View File
@@ -0,0 +1,95 @@
#ifndef DOS_DOSHUNKS_H
#define DOS_DOSHUNKS_H
/*
** $VER: doshunks.h 36.9 (2.6.92)
** Includes Release 40.13
**
** Hunk definitions for object and load modules.
**
** (C) Copyright 1989-1993 Commodore-Amiga, Inc.
** All Rights Reserved
*/
/* hunk types */
#define HUNK_UNIT 999
#define HUNK_NAME 1000
#define HUNK_CODE 1001
#define HUNK_DATA 1002
#define HUNK_BSS 1003
#define HUNK_RELOC32 1004
#define HUNK_ABSRELOC32 HUNK_RELOC32
#define HUNK_RELOC16 1005
#define HUNK_RELRELOC16 HUNK_RELOC16
#define HUNK_RELOC8 1006
#define HUNK_RELRELOC8 HUNK_RELOC8
#define HUNK_EXT 1007
#define HUNK_SYMBOL 1008
#define HUNK_DEBUG 1009
#define HUNK_END 1010
#define HUNK_HEADER 1011
#define HUNK_OVERLAY 1013
#define HUNK_BREAK 1014
#define HUNK_DREL32 1015
#define HUNK_DREL16 1016
#define HUNK_DREL8 1017
#define HUNK_LIB 1018
#define HUNK_INDEX 1019
/*
* Note: V37 LoadSeg uses 1015 (HUNK_DREL32) by mistake. This will continue
* to be supported in future versions, since HUNK_DREL32 is illegal in load files
* anyways. Future versions will support both 1015 and 1020, though anything
* that should be usable under V37 should use 1015.
*/
#define HUNK_RELOC32SHORT 1020
/* see ext_xxx below. New for V39 (note that LoadSeg only handles RELRELOC32).*/
#define HUNK_RELRELOC32 1021
#define HUNK_ABSRELOC16 1022
/*
* Any hunks that have the HUNKB_ADVISORY bit set will be ignored if they
* aren't understood. When ignored, they're treated like HUNK_DEBUG hunks.
* NOTE: this handling of HUNKB_ADVISORY started as of V39 dos.library! If
* lading such executables is attempted under <V39 dos, it will fail with a
* bad hunk type.
*/
#define HUNKB_ADVISORY 29
#define HUNKB_CHIP 30
#define HUNKB_FAST 31
#define HUNKF_ADVISORY (1L<<29)
#define HUNKF_CHIP (1L<<30)
#define HUNKF_FAST (1L<<31)
/* hunk_ext sub-types */
#define EXT_SYMB 0 /* symbol table */
#define EXT_DEF 1 /* relocatable definition */
#define EXT_ABS 2 /* Absolute definition */
#define EXT_RES 3 /* no longer supported */
#define EXT_REF32 129 /* 32 bit absolute reference to symbol */
#define EXT_ABSREF32 EXT_REF32
#define EXT_COMMON 130 /* 32 bit absolute reference to COMMON block */
#define EXT_ABSCOMMON EXT_COMMON
#define EXT_REF16 131 /* 16 bit PC-relative reference to symbol */
#define EXT_RELREF16 EXT_REF16
#define EXT_REF8 132 /* 8 bit PC-relative reference to symbol */
#define EXT_RELREF8 EXT_REF8
#define EXT_DEXT32 133 /* 32 bit data relative reference */
#define EXT_DEXT16 134 /* 16 bit data relative reference */
#define EXT_DEXT8 135 /* 8 bit data relative reference */
/* These are to support some of the '020 and up modes that are rarely used */
#define EXT_RELREF32 136 /* 32 bit PC-relative reference to symbol */
#define EXT_RELCOMMON 137 /* 32 bit PC-relative reference to COMMON block */
/* for completeness... All 680x0's support this */
#define EXT_ABSREF16 138 /* 16 bit absolute reference to symbol */
/* this only exists on '020's and above, in the (d8,An,Xn) address mode */
#define EXT_ABSREF8 139 /* 8 bit absolute reference to symbol */
#endif /* DOS_DOSHUNKS_H */
Binary file not shown.
Binary file not shown.