mirror of
https://github.com/alpine9000/amiga_examples
synced 2025-12-08 14:59:01 +00:00
Added first pass at shrinkler example
This commit is contained in:
@@ -0,0 +1 @@
|
||||
build
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
repo: dffe960c35de060c4e9767c09f66bb1bbc09f06b
|
||||
node: 22f9859cf2ceb9a230d4271a0d905b948d23d43c
|
||||
branch: default
|
||||
latesttag: null
|
||||
latesttagdistance: 57
|
||||
changessincelatesttag: 57
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
Helper classes to access big-endian Amiga words and longwords.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
// A big-endian 16-bit integer with implicit conversions to and from unsigned short
|
||||
class Word {
|
||||
unsigned short value;
|
||||
|
||||
static unsigned short conv(unsigned short val) {
|
||||
#ifndef AMIGA
|
||||
return
|
||||
((val & 0x00ff) << 8) |
|
||||
((val & 0xff00) >> 8);
|
||||
#else
|
||||
return val;
|
||||
#endif
|
||||
}
|
||||
public:
|
||||
Word(unsigned short val) : value(conv(val)) {}
|
||||
Word() : value(0) {}
|
||||
|
||||
operator unsigned short() const {
|
||||
return conv(value);
|
||||
}
|
||||
|
||||
bool operator<(const Word& other) const {
|
||||
return conv(value) < conv(other.value);
|
||||
}
|
||||
|
||||
unsigned short operator+=(unsigned short a) {
|
||||
unsigned short res = conv(value) + a;
|
||||
value = conv(res);
|
||||
return res;
|
||||
}
|
||||
};
|
||||
|
||||
// A big-endian 32-bit integer with implicit conversions to and from unsigned int
|
||||
class Longword {
|
||||
unsigned int value;
|
||||
|
||||
static unsigned int conv(unsigned int val) {
|
||||
#ifndef AMIGA
|
||||
return
|
||||
((val & 0x000000ff) << 24) |
|
||||
((val & 0x0000ff00) << 8) |
|
||||
((val & 0x00ff0000) >> 8) |
|
||||
((val & 0xff000000) >> 24);
|
||||
#else
|
||||
return val;
|
||||
#endif
|
||||
}
|
||||
public:
|
||||
Longword(unsigned int value) : value(conv(value)) {}
|
||||
Longword() : value(0) {}
|
||||
|
||||
operator unsigned int() const {
|
||||
return conv(value);
|
||||
}
|
||||
|
||||
bool operator<(const Longword& other) const {
|
||||
return conv(value) < conv(other.value);
|
||||
}
|
||||
|
||||
unsigned int operator+=(unsigned int a) {
|
||||
unsigned int res = conv(value) + a;
|
||||
value = conv(res);
|
||||
return res;
|
||||
}
|
||||
};
|
||||
Vendored
+35
@@ -0,0 +1,35 @@
|
||||
|
||||
To build Shrinkler natively for your unix-like system, type
|
||||
|
||||
make
|
||||
|
||||
You can cross-build for a particular target using
|
||||
|
||||
make PLATFORM=<platform>
|
||||
|
||||
where <platform> is one of:
|
||||
|
||||
amiga: build for Amiga (build on Cygwin, and see below)
|
||||
windows-32: build for 32-bit Windows (build on Cygwin)
|
||||
windows-64: build for 64-bit Windows (build on Cygwin)
|
||||
native: same as default
|
||||
native-32: native, forced to 32 bits
|
||||
native-64: native, forced to 64 bits
|
||||
|
||||
To build for Amiga, you will first need to download a few things:
|
||||
|
||||
Download
|
||||
http://amiga.sourceforge.net/phps/logger.php?download=GCC-4.5.0-m68k-amigaos-cygwin.7z
|
||||
and unpack it to the toolchain directory.
|
||||
|
||||
Download
|
||||
http://sourceforge.net/projects/amiga/files/Compilers/related/include-20090222.lha
|
||||
and unpack it to the toolchain directory.
|
||||
|
||||
Create the directory toolchain/ixemul-sdk, download
|
||||
http://sourceforge.net/projects/amiga/files/ixemul.library/48.2/ixemul-sdk.lha
|
||||
and unpack it to the directory you just created.
|
||||
|
||||
The Amiga version will need ixemul.library version 48 or later to run.
|
||||
|
||||
Enjoy!
|
||||
Vendored
+115
@@ -0,0 +1,115 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
Abstract interface for entropy coding.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "assert.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
using std::vector;
|
||||
|
||||
class Coder {
|
||||
bool cacheable;
|
||||
bool has_cache;
|
||||
int number_context_offset;
|
||||
int n_number_contexts;
|
||||
vector<vector<unsigned short> > cache;
|
||||
|
||||
protected:
|
||||
Coder() : cacheable(false), has_cache(false)
|
||||
{}
|
||||
|
||||
// Mark coder as cacheable
|
||||
void setCacheable(bool cacheable) {
|
||||
this->cacheable = cacheable;
|
||||
}
|
||||
|
||||
public:
|
||||
// Set parameters for number size cache
|
||||
void setNumberContexts(int number_context_offset, int n_number_contexts, int max_number) {
|
||||
if (!cacheable) return;
|
||||
|
||||
this->number_context_offset = number_context_offset;
|
||||
this->n_number_contexts = n_number_contexts;
|
||||
cache.clear();
|
||||
for (int context_index = 0 ; context_index < n_number_contexts ; context_index++) {
|
||||
int base_context = number_context_offset + (context_index << 8);
|
||||
cache.push_back(vector<unsigned short>());
|
||||
vector<unsigned short>& c = cache.back();
|
||||
c.resize(4);
|
||||
c[2] = code(base_context + 2, 0) + code(base_context + 1, 0);
|
||||
c[3] = code(base_context + 2, 0) + code(base_context + 1, 1);
|
||||
int prev_base = 2;
|
||||
for (int data_bits = 2 ; data_bits < 30 ; data_bits++) {
|
||||
int base = c.size();
|
||||
int base_sizedif = - code(base_context + data_bits * 2 - 2, 0)
|
||||
+ code(base_context + data_bits * 2 - 2, 1)
|
||||
+ code(base_context + data_bits * 2, 0);
|
||||
for (int msb = 0 ; msb <= 1 ; msb++) {
|
||||
int sizedif = base_sizedif + code(base_context + data_bits * 2 - 1, msb);
|
||||
for (int tail = 0 ; tail < 1 << (data_bits - 1) ; tail++) {
|
||||
int size = c[prev_base + tail] + sizedif;
|
||||
c.push_back(size);
|
||||
if (c.size() > max_number) goto next_context;
|
||||
}
|
||||
}
|
||||
prev_base = base;
|
||||
}
|
||||
next_context:;
|
||||
#if 0
|
||||
for (int i = 2 ; i < c.size() ; i++) {
|
||||
assert(c[i] == encodeNumber(base_context, i));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
has_cache = true;
|
||||
}
|
||||
|
||||
// Number of fractional bits in the bit sizes returned by coding functions.
|
||||
static const int BIT_PRECISION = 6;
|
||||
|
||||
// Code the given bit value in the given context.
|
||||
// Returns the coded size of the bit (in fractional bits).
|
||||
virtual int code(int context, int bit) = 0;
|
||||
|
||||
// Encode a number >= 2 using a variable-length encoding.
|
||||
// Returns the coded size of the number (in fractional bits).
|
||||
int encodeNumber(int base_context, int number) {
|
||||
assert(number >= 2);
|
||||
|
||||
if (has_cache) {
|
||||
int context_index = (base_context - number_context_offset) >> 8;
|
||||
vector<unsigned short>& cache_for_context = cache[context_index];
|
||||
if (number < cache_for_context.size()) {
|
||||
return cache_for_context[number];
|
||||
}
|
||||
}
|
||||
|
||||
int size = 0;
|
||||
int context;
|
||||
int i;
|
||||
for (i = 0 ; (4 << i) <= number ; i++) {
|
||||
context = base_context + (i * 2 + 2);
|
||||
size += code(context, 1);
|
||||
}
|
||||
context = base_context + (i * 2 + 2);
|
||||
size += code(context, 0);
|
||||
|
||||
for (; i >= 0 ; i--) {
|
||||
int bit = ((number >> i) & 1);
|
||||
context = base_context + (i * 2 + 1);
|
||||
size += code(context, bit);
|
||||
}
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
virtual ~Coder() {}
|
||||
};
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
A dummy entropy coder which counts the occurrences of symbols for estimating
|
||||
the sizes using a SizeMeasuringCoder during the next compression pass.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
|
||||
using std::vector;
|
||||
|
||||
#include "Coder.h"
|
||||
|
||||
struct ContextCounts {
|
||||
int counts[2];
|
||||
};
|
||||
|
||||
class CountingCoder : public Coder {
|
||||
vector<ContextCounts> context_counts;
|
||||
|
||||
friend class SizeMeasuringCoder;
|
||||
public:
|
||||
CountingCoder(int n_contexts) {
|
||||
struct ContextCounts init_counts = { { 0, 0 } };
|
||||
context_counts.resize(n_contexts, init_counts);
|
||||
}
|
||||
|
||||
CountingCoder(CountingCoder *old_counts, CountingCoder *new_counts) {
|
||||
for (int i = 0 ; i < old_counts->context_counts.size() ; i++) {
|
||||
struct ContextCounts old_count = old_counts->context_counts[i];
|
||||
struct ContextCounts new_count = new_counts->context_counts[i];
|
||||
struct ContextCounts mixed_count = { {
|
||||
(old_count.counts[0] * 3 + new_count.counts[0]) / 4,
|
||||
(old_count.counts[1] * 3 + new_count.counts[1]) / 4
|
||||
} };
|
||||
context_counts.push_back(mixed_count);
|
||||
}
|
||||
}
|
||||
|
||||
virtual int code(int context_index, int bit) {
|
||||
context_counts[context_index].counts[bit]++;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void printRange(FILE *out, int first, int num) {
|
||||
fprintf(out, "[");
|
||||
for (int i = 0 ; i < num ; i++) {
|
||||
if (i > 0) {
|
||||
fprintf(out, " ");
|
||||
}
|
||||
fprintf(out, "%d/%d", context_counts[first + i].counts[0], context_counts[first + i].counts[1]);
|
||||
}
|
||||
fprintf(out, "]");
|
||||
}
|
||||
|
||||
};
|
||||
+239
@@ -0,0 +1,239 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
Cuckoo hash map. Used for mapping offsets to edges in the LZ parser.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <utility>
|
||||
#include <algorithm>
|
||||
#include <new>
|
||||
|
||||
using std::pair;
|
||||
|
||||
template <typename V> class CuckooHash;
|
||||
|
||||
template <typename V>
|
||||
class CuckooHashIterator {
|
||||
const CuckooHash<V>* table;
|
||||
int index;
|
||||
CuckooHashIterator(const CuckooHash<V>* table, int index) : table(table), index(index)
|
||||
{}
|
||||
|
||||
void find() {
|
||||
while (table->element_array[index].first == CuckooHash<V>::UNUSED) index++;
|
||||
}
|
||||
|
||||
friend class CuckooHash<V>;
|
||||
|
||||
public:
|
||||
pair<int, V>& operator*() {
|
||||
find();
|
||||
return table->element_array[index];
|
||||
}
|
||||
|
||||
pair<int, V>* operator->() {
|
||||
find();
|
||||
return &table->element_array[index];
|
||||
}
|
||||
|
||||
CuckooHashIterator<V> operator++(int) {
|
||||
find();
|
||||
return CuckooHashIterator<V>(table, index++);
|
||||
}
|
||||
|
||||
bool operator!=(const CuckooHashIterator<V>& other) {
|
||||
return index != other.index;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename V>
|
||||
class CuckooHash {
|
||||
public:
|
||||
typedef int key_type;
|
||||
typedef pair<key_type, V> value_type;
|
||||
typedef CuckooHashIterator<V> iterator;
|
||||
private:
|
||||
friend class CuckooHashIterator<V>;
|
||||
|
||||
typedef unsigned hash_type;
|
||||
|
||||
static const key_type UNUSED = 0x80000000;
|
||||
static const hash_type HASH1_MUL = 0xF230D3A1;
|
||||
static const hash_type HASH2_MUL = 0x8084027F;
|
||||
static const int INITIAL_SIZE_LOG = 2;
|
||||
|
||||
value_type* element_array;
|
||||
unsigned n_elements:26;
|
||||
unsigned hash_shift:6;
|
||||
|
||||
int array_size() const {
|
||||
return 1 << (sizeof(hash_type) * 8 - hash_shift);
|
||||
}
|
||||
|
||||
void init_array() {
|
||||
int size = array_size();
|
||||
element_array = new value_type[size];
|
||||
for (int i = 0 ; i < size ; i++) {
|
||||
element_array[i].first = UNUSED;
|
||||
element_array[i].second = V();
|
||||
}
|
||||
}
|
||||
|
||||
value_type* get_array() {
|
||||
if (element_array == NULL) {
|
||||
init_array();
|
||||
}
|
||||
return element_array;
|
||||
}
|
||||
|
||||
void init() {
|
||||
n_elements = 0;
|
||||
hash_shift = sizeof(hash_type) * 8 - INITIAL_SIZE_LOG;
|
||||
element_array = NULL;
|
||||
}
|
||||
|
||||
void hashes(key_type key, hash_type& hash1, hash_type& hash2) const {
|
||||
hash_type f = (key << 1) + 1;
|
||||
hash1 = (f * HASH1_MUL) >> hash_shift;
|
||||
hash2 = (f * HASH2_MUL) >> hash_shift;
|
||||
}
|
||||
|
||||
void rehash() {
|
||||
int old_size = array_size();
|
||||
value_type* old_array = get_array();
|
||||
n_elements = 0;
|
||||
hash_shift--;
|
||||
init_array();
|
||||
for (int i = 0 ; i < old_size ; i++) {
|
||||
if (old_array[i].first != UNUSED) {
|
||||
(*this)[old_array[i].first] = old_array[i].second;
|
||||
}
|
||||
}
|
||||
delete[] old_array;
|
||||
}
|
||||
|
||||
void insert(hash_type hash, int key, V value, int n) {
|
||||
value_type* array = get_array();
|
||||
while (array[hash].first != UNUSED) {
|
||||
if (--n < 0) {
|
||||
rehash();
|
||||
(*this)[key] = value;
|
||||
return;
|
||||
}
|
||||
std::swap(key, array[hash].first);
|
||||
std::swap(value, array[hash].second);
|
||||
hash_type hash1;
|
||||
hash_type hash2;
|
||||
hashes(key, hash1, hash2);
|
||||
hash ^= hash1 ^ hash2;
|
||||
}
|
||||
array[hash].first = key;
|
||||
array[hash].second = value;
|
||||
n_elements++;
|
||||
}
|
||||
|
||||
public:
|
||||
CuckooHash() {
|
||||
init();
|
||||
}
|
||||
|
||||
CuckooHash(const CuckooHash& source) {
|
||||
// We only use copy for array initialization, so just create an empty map
|
||||
init();
|
||||
}
|
||||
|
||||
~CuckooHash() {
|
||||
delete[] element_array;
|
||||
}
|
||||
|
||||
void clear() {
|
||||
delete[] element_array;
|
||||
init();
|
||||
}
|
||||
|
||||
iterator begin() const {
|
||||
return CuckooHashIterator<V>(this, 0);
|
||||
}
|
||||
|
||||
iterator end() const {
|
||||
if (element_array == NULL) {
|
||||
// Empty
|
||||
return CuckooHashIterator<V>(this, 0);
|
||||
}
|
||||
|
||||
int index = array_size();
|
||||
assert(element_array != NULL);
|
||||
value_type* array = element_array;
|
||||
while (index > 0 && array[index - 1].first == UNUSED) index--;
|
||||
return CuckooHashIterator<V>(this, index);
|
||||
}
|
||||
|
||||
int size() const {
|
||||
return n_elements;
|
||||
}
|
||||
|
||||
bool empty() const {
|
||||
return size() == 0;
|
||||
}
|
||||
|
||||
int count(int key) const {
|
||||
if (empty()) return 0;
|
||||
|
||||
hash_type hash1;
|
||||
hash_type hash2;
|
||||
hashes(key, hash1, hash2);
|
||||
|
||||
assert(element_array != NULL);
|
||||
value_type* array = element_array;
|
||||
if (array[hash1].first == key || array[hash2].first == key) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void erase(int key) {
|
||||
hash_type hash1;
|
||||
hash_type hash2;
|
||||
hashes(key, hash1, hash2);
|
||||
|
||||
value_type* array = get_array();
|
||||
hash_type hash;
|
||||
if (array[hash1].first == key) {
|
||||
hash = hash1;
|
||||
} else if (array[hash2].first == key) {
|
||||
hash = hash2;
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
array[hash].first = UNUSED;
|
||||
array[hash].second = V();
|
||||
n_elements--;
|
||||
}
|
||||
|
||||
V& operator[](int key) {
|
||||
hash_type hash1;
|
||||
hash_type hash2;
|
||||
hashes(key, hash1, hash2);
|
||||
|
||||
value_type* array = get_array();
|
||||
if (array[hash1].first == key) return array[hash1].second;
|
||||
if (array[hash2].first == key) return array[hash2].second;
|
||||
if (array[hash1].first == UNUSED) {
|
||||
array[hash1].first = key;
|
||||
array[hash1].second = V();
|
||||
n_elements++;
|
||||
return array[hash1].second;
|
||||
}
|
||||
if (array[hash2].first == UNUSED) {
|
||||
array[hash2].first = key;
|
||||
array[hash2].second = V();
|
||||
n_elements++;
|
||||
return array[hash2].second;
|
||||
}
|
||||
insert(hash1, key, V(), n_elements);
|
||||
return (*this)[key];
|
||||
}
|
||||
};
|
||||
|
||||
Vendored
+129
@@ -0,0 +1,129 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
Operations on raw data files, including loading, crunching and saving.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstring>
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <algorithm>
|
||||
|
||||
using std::make_pair;
|
||||
using std::max;
|
||||
using std::min;
|
||||
using std::pair;
|
||||
using std::string;
|
||||
|
||||
#include "AmigaWords.h"
|
||||
#include "Pack.h"
|
||||
#include "RangeDecoder.h"
|
||||
|
||||
class DataFile {
|
||||
vector<unsigned char> data;
|
||||
|
||||
vector<unsigned> compress(PackParams *params, RefEdgeFactory *edge_factory, bool show_progress) {
|
||||
vector<unsigned> pack_buffer;
|
||||
RangeCoder *range_coder = new RangeCoder(LZEncoder::NUM_CONTEXTS + NUM_RELOC_CONTEXTS, pack_buffer);
|
||||
|
||||
// Print compression status header
|
||||
const char *ordinals[] = { "st", "nd", "rd", "th" };
|
||||
printf("Original");
|
||||
for (int p = 1 ; p <= params->iterations ; p++) {
|
||||
printf(" After %d%s pass", p, ordinals[min(p,4)-1]);
|
||||
}
|
||||
printf("\n");
|
||||
|
||||
// Crunch the data
|
||||
range_coder->reset();
|
||||
packData(&data[0], data.size(), 0, params, range_coder, edge_factory, show_progress);
|
||||
range_coder->finish();
|
||||
printf("\n\n");
|
||||
fflush(stdout);
|
||||
|
||||
return pack_buffer;
|
||||
}
|
||||
|
||||
void verify(vector<unsigned>& pack_buffer) {
|
||||
printf("Verifying... ");
|
||||
fflush(stdout);
|
||||
RangeDecoder decoder(LZEncoder::NUM_CONTEXTS + NUM_RELOC_CONTEXTS, pack_buffer);
|
||||
LZDecoder lzd(&decoder);
|
||||
|
||||
// Verify data
|
||||
bool error = false;
|
||||
LZVerifier verifier(0, &data[0], data.size(), data.size());
|
||||
decoder.reset();
|
||||
decoder.setListener(&verifier);
|
||||
if (!lzd.decode(verifier)) {
|
||||
error = true;
|
||||
}
|
||||
|
||||
// Check length
|
||||
if (!error && verifier.size() != data.size()) {
|
||||
printf("Verify error: data has incorrect length (%d, should have been %d)!\n", verifier.size(), (int) data.size());
|
||||
error = true;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
internal_error();
|
||||
}
|
||||
|
||||
printf("OK\n\n");
|
||||
}
|
||||
|
||||
public:
|
||||
void load(const char *filename) {
|
||||
FILE *file;
|
||||
if ((file = fopen(filename, "rb"))) {
|
||||
fseek(file, 0, SEEK_END);
|
||||
int length = ftell(file);
|
||||
fseek(file, 0, SEEK_SET);
|
||||
data.resize(length);
|
||||
if (fread(&data[0], 1, data.size(), file) == data.size()) {
|
||||
fclose(file);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
printf("Error while reading file %s\n\n", filename);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
void save(const char *filename) {
|
||||
FILE *file;
|
||||
if ((file = fopen(filename, "wb"))) {
|
||||
if (fwrite(&data[0], 1, data.size(), file) == data.size()) {
|
||||
fclose(file);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
printf("Error while writing file %s\n\n", filename);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
int size() {
|
||||
return data.size();
|
||||
}
|
||||
|
||||
DataFile* crunch(PackParams *params, RefEdgeFactory *edge_factory, bool show_progress) {
|
||||
vector<unsigned> pack_buffer = compress(params, edge_factory, show_progress);
|
||||
verify(pack_buffer);
|
||||
|
||||
DataFile *ef = new DataFile;
|
||||
ef->data.resize(pack_buffer.size() * 4, 0);
|
||||
|
||||
Longword* dest = (Longword*) (void*) &ef->data[0];
|
||||
for (int i = 0 ; i < pack_buffer.size() ; i++) {
|
||||
dest[i] = pack_buffer[i];
|
||||
}
|
||||
|
||||
return ef;
|
||||
}
|
||||
};
|
||||
Vendored
+38
@@ -0,0 +1,38 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
Abstract interface for entropy decoding.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
class Decoder {
|
||||
public:
|
||||
// Decode a bit in the given context.
|
||||
// Returns the decoded bit value.
|
||||
virtual int decode(int context) = 0;
|
||||
|
||||
// Decode a number >= 2 using a variable-length encoding.
|
||||
// Returns the decoded number.
|
||||
int decodeNumber(int base_context) {
|
||||
int context;
|
||||
int i;
|
||||
for (i = 0 ;; i++) {
|
||||
context = base_context + (i * 2 + 2);
|
||||
if (decode(context) == 0) break;
|
||||
}
|
||||
|
||||
int number = 1;
|
||||
for (; i >= 0 ; i--) {
|
||||
context = base_context + (i * 2 + 1);
|
||||
int bit = decode(context);
|
||||
number = (number << 1) | bit;
|
||||
}
|
||||
|
||||
return number;
|
||||
}
|
||||
|
||||
virtual ~Decoder() {}
|
||||
};
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
Binary Amiga code for the decrunch headers.
|
||||
|
||||
The .dat files are generated from the .bin files by the Makefile.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
unsigned char Header1[] = {
|
||||
#include "Header1.dat"
|
||||
};
|
||||
|
||||
unsigned char Header1T[] = {
|
||||
#include "Header1T.dat"
|
||||
};
|
||||
|
||||
unsigned char Header2[] = {
|
||||
#include "Header2.dat"
|
||||
};
|
||||
|
||||
unsigned char OverlapHeader[] = {
|
||||
#include "OverlapHeader.dat"
|
||||
};
|
||||
|
||||
unsigned char OverlapHeaderT[] = {
|
||||
#include "OverlapHeaderT.dat"
|
||||
};
|
||||
|
||||
unsigned char MiniHeader[] = {
|
||||
#include "MiniHeader.dat"
|
||||
};
|
||||
Vendored
+258
@@ -0,0 +1,258 @@
|
||||
; Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
; auto wb\Header1\Header1_End\
|
||||
; auto wb\Header1T\Header1T_End\
|
||||
; auto wb\Header2\Header2_End\
|
||||
|
||||
INIT_ONE_PROB = $8000
|
||||
ADJUST_SHIFT = 4
|
||||
SINGLE_BIT_CONTEXTS = 1
|
||||
NUM_CONTEXTS = 1536
|
||||
DUMMY_TEXT_LENGTH = 0
|
||||
|
||||
; Exec
|
||||
LIB_VERSION = 20
|
||||
Forbid = -132
|
||||
Permit = -138
|
||||
FreeMem = -210
|
||||
OldOpenLibrary = -408
|
||||
CloseLibrary = -414
|
||||
CacheClearU = -636
|
||||
|
||||
; Dos
|
||||
Write = -48
|
||||
Output = -60
|
||||
|
||||
|
||||
align 0,4
|
||||
Header1:
|
||||
move.l a3,a4
|
||||
move.l (a4)+,d2
|
||||
movem.l d0/a0/a4,-(a7)
|
||||
.hunk: lsl.l #2,d2
|
||||
move.l a4,a5
|
||||
move.l d2,a4
|
||||
move.l (a4)+,d2
|
||||
bne.b .hunk
|
||||
|
||||
move.l $4.w,a6
|
||||
jmp (a4)
|
||||
|
||||
align 0,4
|
||||
Header1_End:
|
||||
|
||||
Header1T:
|
||||
move.l a3,a4
|
||||
move.l (a4)+,d2
|
||||
movem.l d0/a0/a4,-(a7)
|
||||
.hunk: lsl.l #2,d2
|
||||
move.l a4,a5
|
||||
move.l d2,a4
|
||||
move.l (a4)+,d2
|
||||
bne.b .hunk
|
||||
|
||||
TextLengthInstr:
|
||||
move.l #DUMMY_TEXT_LENGTH,d3
|
||||
move.l $4.w,a6
|
||||
lea.l DosName(pc),a1
|
||||
jsr OldOpenLibrary(a6)
|
||||
move.l d0,a6
|
||||
jsr Output(a6)
|
||||
move.l d0,d1
|
||||
beq.b .noout
|
||||
lea.l Header1T_End(pc),a0
|
||||
move.l a0,d2
|
||||
jsr Write(a6)
|
||||
.noout: move.l a6,a1
|
||||
move.l $4.w,a6
|
||||
jsr CloseLibrary(a6)
|
||||
|
||||
jmp (a4)
|
||||
|
||||
DosName:
|
||||
dc.b "dos.library",0
|
||||
|
||||
align 0,4
|
||||
Header1T_End:
|
||||
|
||||
Header2:
|
||||
; Detach last hunk
|
||||
clr.l -(a5)
|
||||
|
||||
; Packed data Start
|
||||
ContextOffsetInstr:
|
||||
lea.l Header2_End(pc),a4
|
||||
|
||||
; Init range decoder state
|
||||
moveq.l #1,d1
|
||||
ror.l #1,d1
|
||||
moveq.l #1,d3
|
||||
; Lowest bit of D2 = 0
|
||||
|
||||
move.l a3,a2
|
||||
HunkLoop:
|
||||
move.l a2,a1
|
||||
addq.l #4,a1
|
||||
; A1 = Hunk Data Destination
|
||||
|
||||
moveq.l #NUM_CONTEXTS>>4,d6
|
||||
lsl.l #4,d6
|
||||
.init: move.w #INIT_ONE_PROB,-(a7)
|
||||
subq.w #1,d6
|
||||
bne.b .init
|
||||
|
||||
; moveq.l #0,d6
|
||||
.lit:
|
||||
addq.b #1,d6
|
||||
.getlit:
|
||||
bsr.b GetBit
|
||||
addx.b d6,d6
|
||||
bcc.b .getlit
|
||||
move.b d6,(a1)+
|
||||
.switch:
|
||||
bsr.b GetKind
|
||||
bcc.b .lit
|
||||
.ref:
|
||||
moveq.l #-1,d6
|
||||
bsr.b GetBit
|
||||
bcs.b .sameoffset
|
||||
.newref:
|
||||
moveq.l #3,d6
|
||||
bsr.b GetNumber
|
||||
moveq.l #2,d5
|
||||
sub.l d7,d5
|
||||
beq.b .hunkend
|
||||
.sameoffset:
|
||||
moveq.l #4,d6
|
||||
bsr.b GetNumber
|
||||
.copyloop:
|
||||
move.b (a1,d5.l),(a1)+
|
||||
subq.l #1,d7
|
||||
bne.b .copyloop
|
||||
.afterref:
|
||||
bsr.b GetKind
|
||||
bcc.b .lit
|
||||
bra.b .newref
|
||||
.hunkend:
|
||||
|
||||
; Relocs
|
||||
move.l a3,d5
|
||||
RelocHunk:
|
||||
addq.l #4,d5
|
||||
move.l a2,a1
|
||||
.relocloop:
|
||||
moveq.l #5,d6
|
||||
bsr.b GetNumber
|
||||
add.l d7,a1
|
||||
lsr.l #2,d7
|
||||
beq.b NextRelocHunk
|
||||
add.l d5,(a1)
|
||||
bra.b .relocloop
|
||||
|
||||
NextRelocHunk:
|
||||
move.l d5,a1
|
||||
move.l -(a1),d5
|
||||
lsl.l #2,d5
|
||||
bne.b RelocHunk
|
||||
NextHunk:
|
||||
lea.l NUM_CONTEXTS*2(a7),a7
|
||||
|
||||
move.l (a2),d4
|
||||
lsl.l #2,d4
|
||||
move.l d4,a2
|
||||
bne.b HunkLoop
|
||||
End:
|
||||
cmp.w #37,LIB_VERSION(a6)
|
||||
blt.b .not204
|
||||
jsr CacheClearU(a6)
|
||||
.not204:
|
||||
jsr Forbid(a6)
|
||||
lea.l Header2-8(pc),a1
|
||||
move.l (a1),d0
|
||||
jsr FreeMem(a6)
|
||||
movem.l (a7)+,d0/a0
|
||||
jmp Permit(a6)
|
||||
|
||||
|
||||
GetKind:
|
||||
move.l a1,d4
|
||||
moveq.l #1,d6
|
||||
and.l d4,d6
|
||||
lsl.w #8,d6
|
||||
GetBit: bra.b GetBitInner
|
||||
|
||||
GetNumber:
|
||||
; D6 = Number context
|
||||
|
||||
; Out: Number in D7
|
||||
lsl.w #8,d6
|
||||
.numberloop:
|
||||
addq.b #2,d6
|
||||
bsr.b GetBitInner
|
||||
bcs.b .numberloop
|
||||
moveq.l #1,d7
|
||||
subq.b #1,d6
|
||||
.bitsloop:
|
||||
bsr.b GetBitInner
|
||||
addx.l d7,d7
|
||||
subq.b #2,d6
|
||||
bcc.b .bitsloop
|
||||
rts
|
||||
|
||||
; D6 = Bit context
|
||||
|
||||
; D1 = Input bit buffer
|
||||
; D2 = Range value
|
||||
; D3 = Interval size
|
||||
|
||||
; Out: Bit in C and X
|
||||
|
||||
readbit:
|
||||
add.l d1,d1
|
||||
bne.b nonewword
|
||||
move.l (a4)+,d1
|
||||
addx.l d1,d1
|
||||
nonewword:
|
||||
addx.w d2,d2
|
||||
add.w d3,d3
|
||||
GetBitInner:
|
||||
tst.w d3
|
||||
bpl.b readbit
|
||||
|
||||
lea.l 4+SINGLE_BIT_CONTEXTS*2(a7,d6.l),a5
|
||||
add.l d6,a5
|
||||
move.w (a5),d4
|
||||
; D4 = One prob
|
||||
|
||||
lsr.w #ADJUST_SHIFT,d4
|
||||
sub.w d4,(a5)
|
||||
add.w (a5),d4
|
||||
|
||||
mulu.w d3,d4
|
||||
swap.w d4
|
||||
|
||||
sub.w d4,d2
|
||||
blo.b .one
|
||||
.zero:
|
||||
; oneprob = oneprob * (1 - adjust) = oneprob - oneprob * adjust
|
||||
sub.w d4,d3
|
||||
; 0 in C and X
|
||||
rts
|
||||
.one:
|
||||
; onebrob = 1 - (1 - oneprob) * (1 - adjust) = oneprob - oneprob * adjust + adjust
|
||||
add.w #$ffff>>ADJUST_SHIFT,(a5)
|
||||
move.w d4,d3
|
||||
add.w d4,d2
|
||||
; 1 in C and X
|
||||
rts
|
||||
|
||||
align 0,4
|
||||
|
||||
Header2_End:
|
||||
|
||||
printv Header1_End-Header1
|
||||
printv Header1T_End-Header1T
|
||||
printv Header2_End-Header2
|
||||
printt
|
||||
printv TextLengthInstr+2-Header1T
|
||||
printv ContextOffsetInstr+2-Header2
|
||||
Vendored
BIN
Binary file not shown.
Vendored
+1
@@ -0,0 +1 @@
|
||||
0x28, 0x4B, 0x24, 0x1C, 0x48, 0xE7, 0x80, 0x88, 0xE5, 0x8A, 0x2A, 0x4C, 0x28, 0x42, 0x24, 0x1C, 0x66, 0xF6, 0x2C, 0x78, 0x00, 0x04, 0x4E, 0xD4,
|
||||
BIN
Binary file not shown.
+1
@@ -0,0 +1 @@
|
||||
0x28, 0x4B, 0x24, 0x1C, 0x48, 0xE7, 0x80, 0x88, 0xE5, 0x8A, 0x2A, 0x4C, 0x28, 0x42, 0x24, 0x1C, 0x66, 0xF6, 0x26, 0x3C, 0x00, 0x00, 0x00, 0x00, 0x2C, 0x78, 0x00, 0x04, 0x43, 0xFA, 0x00, 0x26, 0x4E, 0xAE, 0xFE, 0x68, 0x2C, 0x40, 0x4E, 0xAE, 0xFF, 0xC4, 0x22, 0x00, 0x67, 0x0A, 0x41, 0xFA, 0x00, 0x20, 0x24, 0x08, 0x4E, 0xAE, 0xFF, 0xD0, 0x22, 0x4E, 0x2C, 0x78, 0x00, 0x04, 0x4E, 0xAE, 0xFE, 0x62, 0x4E, 0xD4, 0x64, 0x6F, 0x73, 0x2E, 0x6C, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x00,
|
||||
Vendored
BIN
Binary file not shown.
Vendored
+1
@@ -0,0 +1 @@
|
||||
0x42, 0xA5, 0x49, 0xFA, 0x00, 0xE8, 0x72, 0x01, 0xE2, 0x99, 0x76, 0x01, 0x24, 0x4B, 0x22, 0x4A, 0x58, 0x89, 0x7C, 0x60, 0xE9, 0x8E, 0x3F, 0x3C, 0x80, 0x00, 0x53, 0x46, 0x66, 0xF8, 0x52, 0x06, 0x61, 0x7E, 0xDD, 0x06, 0x64, 0xFA, 0x12, 0xC6, 0x61, 0x6E, 0x64, 0xF2, 0x7C, 0xFF, 0x61, 0x70, 0x65, 0x0A, 0x7C, 0x03, 0x61, 0x6C, 0x7A, 0x02, 0x9A, 0x87, 0x67, 0x12, 0x7C, 0x04, 0x61, 0x62, 0x12, 0xF1, 0x58, 0x00, 0x53, 0x87, 0x66, 0xF8, 0x61, 0x4E, 0x64, 0xD2, 0x60, 0xE4, 0x2A, 0x0B, 0x58, 0x85, 0x22, 0x4A, 0x7C, 0x05, 0x61, 0x4A, 0xD3, 0xC7, 0xE4, 0x8F, 0x67, 0x04, 0xDB, 0x91, 0x60, 0xF2, 0x22, 0x45, 0x2A, 0x21, 0xE5, 0x8D, 0x66, 0xE6, 0x4F, 0xEF, 0x0C, 0x00, 0x28, 0x12, 0xE5, 0x8C, 0x24, 0x44, 0x66, 0x98, 0x0C, 0x6E, 0x00, 0x25, 0x00, 0x14, 0x6D, 0x04, 0x4E, 0xAE, 0xFD, 0x84, 0x4E, 0xAE, 0xFF, 0x7C, 0x43, 0xFA, 0xFF, 0x70, 0x20, 0x11, 0x4E, 0xAE, 0xFF, 0x2E, 0x4C, 0xDF, 0x01, 0x01, 0x4E, 0xEE, 0xFF, 0x76, 0x28, 0x09, 0x7C, 0x01, 0xCC, 0x84, 0xE1, 0x4E, 0x60, 0x22, 0xE1, 0x4E, 0x54, 0x06, 0x61, 0x1C, 0x65, 0xFA, 0x7E, 0x01, 0x53, 0x06, 0x61, 0x14, 0xDF, 0x87, 0x55, 0x06, 0x64, 0xF8, 0x4E, 0x75, 0xD2, 0x81, 0x66, 0x04, 0x22, 0x1C, 0xD3, 0x81, 0xD5, 0x42, 0xD6, 0x43, 0x4A, 0x43, 0x6A, 0xF0, 0x4B, 0xF7, 0x68, 0x06, 0xDB, 0xC6, 0x38, 0x15, 0xE8, 0x4C, 0x99, 0x55, 0xD8, 0x55, 0xC8, 0xC3, 0x48, 0x44, 0x94, 0x44, 0x65, 0x04, 0x96, 0x44, 0x4E, 0x75, 0x06, 0x55, 0x0F, 0xFF, 0x36, 0x04, 0xD4, 0x44, 0x4E, 0x75,
|
||||
Vendored
+94
@@ -0,0 +1,94 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
Heap-based priority queue with removal support.
|
||||
|
||||
The element type must have an accessible _heap_index integer field.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <functional>
|
||||
|
||||
using std::vector;
|
||||
using std::less;
|
||||
|
||||
template <class T>
|
||||
class Heap {
|
||||
vector<T> elements;
|
||||
less<T> compare;
|
||||
|
||||
void swap(int i1, int i2) {
|
||||
T t1 = elements[i1];
|
||||
T t2 = elements[i2];
|
||||
elements[i1] = t2;
|
||||
elements[i2] = t1;
|
||||
t2->_heap_index = i1;
|
||||
t1->_heap_index = i2;
|
||||
}
|
||||
|
||||
void up(int i) {
|
||||
while (i > 0) {
|
||||
int pi = (i-1)/2;
|
||||
if (!compare(elements[pi], elements[i])) return;
|
||||
swap(i, pi);
|
||||
i = pi;
|
||||
}
|
||||
}
|
||||
|
||||
void down(int i) {
|
||||
while (i*2+1 < elements.size()) {
|
||||
int ci1 = i*2+1;
|
||||
int ci2 = i*2+2;
|
||||
int ci = ci2 < elements.size() && compare(elements[ci1], elements[ci2]) ? ci2 : ci1;
|
||||
if (!compare(elements[i], elements[ci])) return;
|
||||
swap(i, ci);
|
||||
i = ci;
|
||||
}
|
||||
}
|
||||
|
||||
T remove_index(int i) {
|
||||
T removed = elements[i];
|
||||
T last = elements[elements.size()-1];
|
||||
elements[i] = last;
|
||||
elements.pop_back();
|
||||
last->_heap_index = i;
|
||||
down(i);
|
||||
return removed;
|
||||
}
|
||||
|
||||
public:
|
||||
Heap() {}
|
||||
|
||||
void insert(T t) {
|
||||
elements.push_back(t);
|
||||
t->_heap_index = elements.size()-1;
|
||||
up(elements.size()-1);
|
||||
}
|
||||
|
||||
void remove(T t) {
|
||||
if (contains(t)) {
|
||||
remove_index(t->_heap_index);
|
||||
}
|
||||
}
|
||||
|
||||
T remove_largest() {
|
||||
return remove_index(0);
|
||||
}
|
||||
|
||||
bool contains(T t) {
|
||||
return t->_heap_index < elements.size() && elements[t->_heap_index] == t;
|
||||
}
|
||||
|
||||
int size() {
|
||||
return elements.size();
|
||||
}
|
||||
|
||||
void clear() {
|
||||
elements.clear();
|
||||
}
|
||||
|
||||
};
|
||||
Vendored
+895
@@ -0,0 +1,895 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
Operations on Amiga executables, including loading, parsing,
|
||||
hunk merging, crunching and saving.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstring>
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <algorithm>
|
||||
|
||||
using std::make_pair;
|
||||
using std::max;
|
||||
using std::min;
|
||||
using std::pair;
|
||||
using std::string;
|
||||
|
||||
#include "doshunks.h"
|
||||
#include "AmigaWords.h"
|
||||
#include "DecrunchHeaders.h"
|
||||
#include "Pack.h"
|
||||
#include "RangeDecoder.h"
|
||||
#include "LZDecoder.h"
|
||||
|
||||
const char *hunktype[HUNK_ABSRELOC16-HUNK_UNIT+1] = {
|
||||
"UNIT","NAME","CODE","DATA","BSS ","RELOC32","RELOC16","RELOC8",
|
||||
"EXT","SYMBOL","DEBUG","END","HEADER","","OVERLAY","BREAK",
|
||||
"DREL32","DREL16","DREL8","LIB","INDEX",
|
||||
"RELOC32SHORT","RELRELOC32","ABSRELOC16"
|
||||
};
|
||||
|
||||
#define HUNKF_MASK (HUNKF_FAST | HUNKF_CHIP)
|
||||
#define NUM_RELOC_CONTEXTS 256
|
||||
|
||||
class HunkInfo {
|
||||
public:
|
||||
HunkInfo() { type = 0; relocentries = 0; }
|
||||
|
||||
unsigned type; // HUNK_<type>
|
||||
unsigned flags; // HUNKF_<flag>
|
||||
int memsize,datasize; // longwords
|
||||
int datastart; // longword index in file
|
||||
int relocstart; // longword index in file
|
||||
int relocentries; // no. of entries
|
||||
};
|
||||
|
||||
// Compare waste space
|
||||
class HunkMergeCompare {
|
||||
vector<HunkInfo>& hunks;
|
||||
|
||||
int waste(int h) {
|
||||
if (hunks[h].type == HUNK_BSS) {
|
||||
return hunks[h].memsize;
|
||||
} else {
|
||||
return hunks[h].memsize - hunks[h].datasize;
|
||||
}
|
||||
}
|
||||
public:
|
||||
HunkMergeCompare(vector<HunkInfo>& hunks) : hunks(hunks) {}
|
||||
|
||||
bool operator()(int h1, int h2) {
|
||||
return waste(h1) < waste(h2);
|
||||
}
|
||||
};
|
||||
|
||||
class LZVerifier : public LZReceiver, public CompressedDataReadListener {
|
||||
int hunk;
|
||||
unsigned char *data;
|
||||
int data_length;
|
||||
int hunk_mem;
|
||||
int pos;
|
||||
|
||||
unsigned char getData(int i) {
|
||||
if (data == NULL || i >= data_length) return 0;
|
||||
return data[i];
|
||||
}
|
||||
|
||||
public:
|
||||
int compressed_longword_count;
|
||||
int front_overlap_margin;
|
||||
|
||||
LZVerifier(int hunk, unsigned char *data, int data_length, int hunk_mem) : hunk(hunk), data(data), data_length(data_length), hunk_mem(hunk_mem), pos(0) {
|
||||
compressed_longword_count = 0;
|
||||
front_overlap_margin = 0;
|
||||
}
|
||||
|
||||
bool receiveLiteral(unsigned char lit) {
|
||||
if (pos >= hunk_mem) {
|
||||
printf("Verify error: literal at position %d in hunk %d overflows hunk!\n",
|
||||
pos, hunk);
|
||||
return false;
|
||||
}
|
||||
if (lit != getData(pos)) {
|
||||
printf("Verify error: literal at position %d in hunk %d has incorrect value (0x%02X, should be 0x%02X)!\n",
|
||||
pos, hunk, lit, getData(pos));
|
||||
return false;
|
||||
}
|
||||
pos += 1;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool receiveReference(int offset, int length) {
|
||||
if (offset < 1 || offset > pos) {
|
||||
printf("Verify error: reference at position %d in hunk %d has invalid offset (%d)!\n",
|
||||
pos, hunk, offset);
|
||||
return false;
|
||||
}
|
||||
if (length > hunk_mem - pos) {
|
||||
printf("Verify error: reference at position %d in hunk %d overflows hunk (length %d, %d bytes past end)!\n",
|
||||
pos, hunk, length, pos + length - hunk_mem);
|
||||
return false;
|
||||
}
|
||||
for (int i = 0 ; i < length ; i++) {
|
||||
if (getData(pos - offset + i) != getData(pos + i)) {
|
||||
printf("Verify error: reference at position %d in hunk %d has incorrect value for byte %d of %d (0x%02X, should be 0x%02X)!\n",
|
||||
pos, hunk, i, length, getData(pos - offset + i), getData(pos + i));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
pos += length;
|
||||
return true;
|
||||
}
|
||||
|
||||
int size() {
|
||||
return pos;
|
||||
}
|
||||
|
||||
void read(int index) {
|
||||
// Another longword of compresed data read
|
||||
int margin = pos - compressed_longword_count * 4;
|
||||
if (margin > front_overlap_margin) {
|
||||
front_overlap_margin = margin;
|
||||
}
|
||||
compressed_longword_count += 1;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
class HunkFile {
|
||||
vector<Longword> data;
|
||||
vector<HunkInfo> hunks;
|
||||
|
||||
vector<unsigned> compress_hunks(PackParams *params, bool overlap, bool mini, RefEdgeFactory *edge_factory, bool show_progress) {
|
||||
int numhunks = hunks.size();
|
||||
|
||||
vector<unsigned> pack_buffer;
|
||||
RangeCoder *range_coder = new RangeCoder(LZEncoder::NUM_CONTEXTS + NUM_RELOC_CONTEXTS, pack_buffer);
|
||||
|
||||
// Print compression status header
|
||||
const char *ordinals[] = { "st", "nd", "rd", "th" };
|
||||
printf("Hunk Original");
|
||||
for (int p = 1 ; p <= params->iterations ; p++) {
|
||||
printf(" After %d%s pass", p, ordinals[min(p,4)-1]);
|
||||
}
|
||||
if (!mini) {
|
||||
printf(" Relocs");
|
||||
}
|
||||
printf("\n");
|
||||
|
||||
// Crunch the hunks, one by one.
|
||||
for (int h = 0 ; h < (mini ? 1 : numhunks) ; h++) {
|
||||
printf("%4d ", h);
|
||||
range_coder->reset();
|
||||
switch (hunks[h].type) {
|
||||
case HUNK_CODE:
|
||||
case HUNK_DATA:
|
||||
{
|
||||
// Pack data
|
||||
unsigned char *hunk_data = (unsigned char *) &data[hunks[h].datastart];
|
||||
int hunk_data_length = hunks[h].datasize * 4;
|
||||
// Trim trailing zeros
|
||||
while (hunk_data_length > 0 && hunk_data[hunk_data_length - 1] == 0) {
|
||||
hunk_data_length--;
|
||||
}
|
||||
int zero_padding = mini ? 0 : hunks[h].memsize * 4 - hunk_data_length;
|
||||
packData(hunk_data, hunk_data_length, zero_padding, params, range_coder, edge_factory, show_progress);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
int zero_padding = mini ? 0 : hunks[h].memsize * 4;
|
||||
packData(NULL, 0, zero_padding, params, range_coder, edge_factory, show_progress);
|
||||
break;
|
||||
}
|
||||
|
||||
if (!mini) {
|
||||
// Reloc table
|
||||
int reloc_size = 0;
|
||||
for (int rh = 0 ; rh < numhunks ; rh++) {
|
||||
vector<int> offsets;
|
||||
if (hunks[h].relocentries > 0) {
|
||||
int spos = hunks[h].relocstart;
|
||||
while (data[spos] != 0) {
|
||||
int rn = data[spos++];
|
||||
if (data[spos++] == rh) {
|
||||
while (rn--) {
|
||||
offsets.push_back(data[spos++]);
|
||||
}
|
||||
} else {
|
||||
spos += rn;
|
||||
}
|
||||
}
|
||||
sort(offsets.begin(), offsets.end());
|
||||
}
|
||||
int last_offset = -4;
|
||||
for (int ri = 0 ; ri < offsets.size() ; ri++) {
|
||||
int offset = offsets[ri];
|
||||
int delta = offset - last_offset;
|
||||
if (delta < 4) {
|
||||
printf("\n\nError in input file: overlapping reloc entries.\n\n");
|
||||
exit(1);
|
||||
}
|
||||
reloc_size += range_coder->encodeNumber(LZEncoder::NUM_CONTEXTS, delta);
|
||||
last_offset = offset;
|
||||
}
|
||||
reloc_size += range_coder->encodeNumber(LZEncoder::NUM_CONTEXTS, 2);
|
||||
}
|
||||
printf(" %10.3f", reloc_size / (double) (8 << Coder::BIT_PRECISION));
|
||||
}
|
||||
printf("\n");
|
||||
fflush(stdout);
|
||||
}
|
||||
range_coder->finish();
|
||||
printf("\n");
|
||||
|
||||
return pack_buffer;
|
||||
}
|
||||
|
||||
vector<pair<int,int> > verify(vector<unsigned>& pack_buffer, bool overlap, bool mini) {
|
||||
int numhunks = hunks.size();
|
||||
vector<pair<int,int> > count_and_hunksize;
|
||||
|
||||
printf("Verifying... ");
|
||||
fflush(stdout);
|
||||
RangeDecoder decoder(LZEncoder::NUM_CONTEXTS + NUM_RELOC_CONTEXTS, pack_buffer);
|
||||
LZDecoder lzd(&decoder);
|
||||
for (int h = 0 ; h < (mini ? 1 : numhunks) ; h++) {
|
||||
unsigned char *hunk_data;
|
||||
int hunk_data_length = hunks[h].datasize * 4;
|
||||
if (hunks[h].type != HUNK_BSS) {
|
||||
// Find hunk data
|
||||
hunk_data = (unsigned char *) &data[hunks[h].datastart];
|
||||
if (mini) {
|
||||
// Trim trailing zeros
|
||||
while (hunk_data_length > 0 && hunk_data[hunk_data_length - 1] == 0) {
|
||||
hunk_data_length--;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Signal empty hunk by NULL data pointer
|
||||
hunk_data = NULL;
|
||||
}
|
||||
|
||||
// Verify data
|
||||
bool error = false;
|
||||
LZVerifier verifier(h, hunk_data, hunk_data_length, hunks[h].memsize * sizeof(Longword));
|
||||
decoder.reset();
|
||||
decoder.setListener(&verifier);
|
||||
if (!lzd.decode(verifier)) {
|
||||
error = true;
|
||||
}
|
||||
|
||||
// Check length
|
||||
if (!error && !mini && verifier.size() != hunks[h].memsize * sizeof(Longword)) {
|
||||
printf("Verify error: hunk %d has incorrect length (%d, should have been %d)!\n", h, verifier.size(), hunk_data_length);
|
||||
error = true;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
internal_error();
|
||||
}
|
||||
|
||||
if (!mini) {
|
||||
// Skip relocs
|
||||
for (int rh = 0 ; rh < numhunks ; rh++) {
|
||||
int delta;
|
||||
do {
|
||||
delta = decoder.decodeNumber(LZEncoder::NUM_CONTEXTS);
|
||||
} while (delta != 2);
|
||||
}
|
||||
}
|
||||
|
||||
int margin = verifier.front_overlap_margin;
|
||||
int count = verifier.compressed_longword_count;
|
||||
int min_hunksize = (margin == 0 ? 1 : (margin + 3) / 4) + count;
|
||||
count_and_hunksize.push_back(make_pair(count, min_hunksize));
|
||||
}
|
||||
printf("OK\n\n");
|
||||
|
||||
return count_and_hunksize;
|
||||
}
|
||||
|
||||
public:
|
||||
void load(const char *filename) {
|
||||
FILE *file;
|
||||
if ((file = fopen(filename, "rb"))) {
|
||||
fseek(file, 0, SEEK_END);
|
||||
int length = ftell(file);
|
||||
fseek(file, 0, SEEK_SET);
|
||||
if (length & 3) {
|
||||
printf("File %s has an illegal size!\n\n", filename);
|
||||
fclose(file);
|
||||
exit(1);
|
||||
}
|
||||
data.resize(length / 4);
|
||||
if (fread(&data[0], 4, data.size(), file) == data.size()) {
|
||||
fclose(file);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
printf("Error while reading file %s\n\n", filename);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
void save(const char *filename) {
|
||||
FILE *file;
|
||||
if ((file = fopen(filename, "wb"))) {
|
||||
if (fwrite(&data[0], 4, data.size(), file) == data.size()) {
|
||||
fclose(file);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
printf("Error while writing file %s\n\n", filename);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
int size() {
|
||||
return data.size() * 4;
|
||||
}
|
||||
|
||||
bool analyze() {
|
||||
int index = 0;
|
||||
int length = data.size();
|
||||
|
||||
if (data[index++] != HUNK_HEADER) {
|
||||
printf("No hunk header!\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
while (data[index++]) {
|
||||
index += data[index];
|
||||
if (index >= length) {
|
||||
printf("Bad hunk header!\n");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
int numhunks = data[index++];
|
||||
if (numhunks == 0) {
|
||||
printf("No hunks!\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (data[index++] != 0 || data[index++] != numhunks-1) {
|
||||
printf("Unsupported hunk load limits!\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
hunks.resize(numhunks);
|
||||
for (int h = 0 ; h < numhunks ; h++) {
|
||||
hunks[h].memsize = data[index] & 0x0fffffff;
|
||||
switch (hunks[h].flags = data[index] & 0xf0000000) {
|
||||
case 0:
|
||||
case HUNKF_CHIP:
|
||||
case HUNKF_FAST:
|
||||
break;
|
||||
default:
|
||||
printf("Illegal hunk flags!\n");
|
||||
return false;
|
||||
}
|
||||
index++;
|
||||
}
|
||||
|
||||
// Parse hunks
|
||||
printf("Hunk Mem Type Mem size Data size Data sum Relocs\n");
|
||||
for (int h = 0, nh = 0 ; h < numhunks ;) {
|
||||
unsigned flags = hunks[h].flags, type;
|
||||
int hunk_length, symlen, n_symbols;
|
||||
int lh = h;
|
||||
printf("%4d %s ", h, flags == HUNKF_CHIP ? "CHIP" : flags == HUNKF_FAST ? "FAST" : "ANY ");
|
||||
int missing_relocs = 0;
|
||||
const char *note = "";
|
||||
while (lh == h) {
|
||||
if (index >= length) {
|
||||
printf("\nUnexpected end of file!\n");
|
||||
return false;
|
||||
}
|
||||
type = data[index++] & 0x0fffffff;
|
||||
if (index >= length && type != HUNK_END) {
|
||||
printf("\nUnexpected end of file!\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (missing_relocs && type != HUNK_RELOC32) {
|
||||
printf(" %s\n", note);
|
||||
note = "";
|
||||
missing_relocs = 0;
|
||||
}
|
||||
switch (type) {
|
||||
case HUNK_UNIT:
|
||||
case HUNK_NAME:
|
||||
case HUNK_DEBUG:
|
||||
printf(" %s (skipped)\n",hunktype[type-HUNK_UNIT]);
|
||||
hunk_length = data[index++];
|
||||
index += hunk_length;
|
||||
break;
|
||||
case HUNK_SYMBOL:
|
||||
n_symbols = 0;
|
||||
symlen = data[index++];
|
||||
while (symlen > 0) {
|
||||
n_symbols++;
|
||||
index += symlen+1;
|
||||
symlen = data[index++];
|
||||
}
|
||||
printf(" SYMBOL (%d entries)\n", n_symbols);
|
||||
break;
|
||||
case HUNK_CODE:
|
||||
case HUNK_DATA:
|
||||
case HUNK_BSS:
|
||||
if (nh > h) {
|
||||
h = nh;
|
||||
index--;
|
||||
break;
|
||||
}
|
||||
hunks[h].type = type;
|
||||
hunks[h].datasize = data[index++];
|
||||
printf("%4s%10d %10d", hunktype[type-HUNK_UNIT], hunks[h].memsize*4, hunks[h].datasize*4);
|
||||
if (type != HUNK_BSS) {
|
||||
hunks[h].datastart = index;
|
||||
index += hunks[h].datasize;
|
||||
if (hunks[h].datasize > 0) {
|
||||
int sum = 0;
|
||||
for (int pos = hunks[h].datastart ; pos < hunks[h].datastart+hunks[h].datasize ; pos++) {
|
||||
sum += data[pos];
|
||||
}
|
||||
printf(" %08x", sum);
|
||||
} else {
|
||||
printf(" ");
|
||||
}
|
||||
}
|
||||
if (hunks[h].datasize > hunks[h].memsize) {
|
||||
note = " Hunk size overflow corrected!";
|
||||
hunks[h].memsize = hunks[h].datasize;
|
||||
}
|
||||
nh = h+1;
|
||||
missing_relocs = 1;
|
||||
break;
|
||||
case HUNK_RELOC32:
|
||||
hunks[h].relocstart = index;
|
||||
{
|
||||
int n,tot = 0;
|
||||
while ((n = data[index++]) != 0) {
|
||||
if (n < 0 || index+n+2 >= length || data[index++] >= numhunks) {
|
||||
printf("\nError in reloc table!\n");
|
||||
return false;
|
||||
}
|
||||
tot += n;
|
||||
while (n--) {
|
||||
if (data[index++] > hunks[h].memsize*4-4) {
|
||||
printf("\nError in reloc table!\n");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
hunks[h].relocentries = tot;
|
||||
printf(" %6d%s\n", tot, note);
|
||||
note = "";
|
||||
missing_relocs = 0;
|
||||
}
|
||||
break;
|
||||
case HUNK_END:
|
||||
if (hunks[h].type == 0) {
|
||||
printf("Empty%9d\n", hunks[h].memsize*4);
|
||||
return false;
|
||||
}
|
||||
h = h+1; nh = h;
|
||||
break;
|
||||
case HUNK_RELOC16:
|
||||
case HUNK_RELOC8:
|
||||
case HUNK_EXT:
|
||||
case HUNK_HEADER:
|
||||
case HUNK_OVERLAY:
|
||||
case HUNK_BREAK:
|
||||
case HUNK_DREL32:
|
||||
case HUNK_DREL16:
|
||||
case HUNK_DREL8:
|
||||
case HUNK_LIB:
|
||||
case HUNK_INDEX:
|
||||
case HUNK_RELOC32SHORT:
|
||||
case HUNK_RELRELOC32:
|
||||
case HUNK_ABSRELOC16:
|
||||
printf("%s (unsupported)\n",hunktype[type-HUNK_UNIT]);
|
||||
return false;
|
||||
default:
|
||||
printf("Unknown (%08X)\n",type);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (index < length) {
|
||||
printf("Warning: %d bytes of extra data at the end of the file!\n", (length-index)*4);
|
||||
}
|
||||
printf("\n");
|
||||
return true;
|
||||
}
|
||||
|
||||
int memory_usage(bool include_last_hunk) {
|
||||
int sum = 0;
|
||||
int hunks_to_sum = include_last_hunk ? hunks.size() : hunks.size() - 1;
|
||||
for (int h = 0 ; h < hunks_to_sum ; h++) {
|
||||
sum += ((hunks[h].memsize * 4 + 4) & -8) + 8;
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
vector<pair<unsigned, vector<int> > > merged_hunklist() {
|
||||
int numhunks = hunks.size();
|
||||
vector<pair<unsigned, vector<int> > > hunklist(3);
|
||||
unsigned flags0 = hunks[0].flags;
|
||||
unsigned flags1 = (~flags0) & HUNKF_CHIP;
|
||||
unsigned flags2 = HUNKF_CHIP + HUNKF_FAST - flags0 - flags1;
|
||||
hunklist[0].first = flags0 | HUNK_CODE;
|
||||
hunklist[1].first = flags1 | HUNK_CODE;
|
||||
hunklist[2].first = flags2 | HUNK_CODE;
|
||||
|
||||
HunkMergeCompare comp(hunks);
|
||||
|
||||
// Go through the 3 resulting hunks, one for each memory type.
|
||||
for (int dh = 0 ; dh < 3 ; dh++) {
|
||||
for (int sh = 0 ; sh < numhunks ; sh++) {
|
||||
if (hunks[sh].flags == (hunklist[dh].first & HUNKF_MASK)) {
|
||||
hunklist[dh].second.push_back(sh);
|
||||
}
|
||||
}
|
||||
stable_sort(hunklist[dh].second.begin(), hunklist[dh].second.end(), comp);
|
||||
}
|
||||
|
||||
// Remove unused memory types
|
||||
vector<pair<unsigned, vector<int> > > result;
|
||||
for (int dh = 0 ; dh < 3 ; dh++) {
|
||||
if (hunklist[dh].second.size() > 0) {
|
||||
result.push_back(hunklist[dh]);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
HunkFile* merge_hunks(const vector<pair<unsigned, vector<int> > >& hunklist) {
|
||||
int numhunks = hunks.size();
|
||||
int dnh = hunklist.size();
|
||||
int bufsize = data.size()+3; // Reloc can write 3 further temporarily.
|
||||
|
||||
// Calculate safe size of new file buffer
|
||||
for (int dh = 0 ; dh < dnh ; dh++) {
|
||||
int waste = 0;
|
||||
for (int shi = 0 ; shi < hunklist[dh].second.size() ; shi++) {
|
||||
int sh = hunklist[dh].second[shi];
|
||||
if (hunks[sh].type != HUNK_BSS) {
|
||||
bufsize += waste;
|
||||
waste = -hunks[sh].datasize;
|
||||
}
|
||||
waste += hunks[sh].memsize;
|
||||
}
|
||||
}
|
||||
|
||||
// Processed file
|
||||
HunkFile *ef = new HunkFile;
|
||||
ef->data.resize(bufsize, 0);
|
||||
ef->hunks.resize(dnh);
|
||||
|
||||
vector<int> dhunk(numhunks);
|
||||
vector<int> offset(numhunks);
|
||||
|
||||
// Find destination hunk and offset for all source hunks.
|
||||
for (int dh = 0 ; dh < dnh ; dh++) {
|
||||
unsigned hunkf = hunklist[dh].first;
|
||||
ef->hunks[dh].type = hunkf & 0x0fffffff;
|
||||
ef->hunks[dh].flags = hunkf & 0xf0000000;
|
||||
int memsize = 0;
|
||||
int datasize = 0;
|
||||
for (int shi = 0 ; shi < hunklist[dh].second.size() ; shi++) {
|
||||
int sh = hunklist[dh].second[shi];
|
||||
memsize = (memsize+1)&-2;
|
||||
dhunk[sh] = dh;
|
||||
offset[sh] = memsize*4;
|
||||
if (hunks[sh].type != HUNK_BSS) {
|
||||
datasize = memsize + hunks[sh].datasize;
|
||||
}
|
||||
memsize += hunks[sh].memsize;
|
||||
}
|
||||
ef->hunks[dh].memsize = memsize;
|
||||
ef->hunks[dh].datasize = datasize;
|
||||
}
|
||||
|
||||
// Write new hunk header
|
||||
int dpos = 0;
|
||||
ef->data[dpos++] = HUNK_HEADER;
|
||||
ef->data[dpos++] = 0;
|
||||
ef->data[dpos++] = ef->hunks.size();
|
||||
ef->data[dpos++] = 0;
|
||||
ef->data[dpos++] = ef->hunks.size()-1;
|
||||
for (int dh = 0 ; dh < ef->hunks.size() ; dh++) {
|
||||
ef->data[dpos++] = ef->hunks[dh].memsize | ef->hunks[dh].flags;
|
||||
}
|
||||
|
||||
// Generate new hunks
|
||||
for (int dh = 0 ; dh < dnh ; dh++) {
|
||||
// Put hunk type and data (or bss) size.
|
||||
ef->data[dpos++] = ef->hunks[dh].type;
|
||||
ef->data[dpos++] = ef->hunks[dh].datasize;
|
||||
ef->hunks[dh].datastart = dpos;
|
||||
|
||||
// Run through the implied source hunks.
|
||||
int hoffset = 0;
|
||||
for (int shi = 0 ; shi < hunklist[dh].second.size() ; shi++) {
|
||||
int sh = hunklist[dh].second[shi];
|
||||
if (hunks[sh].type != HUNK_BSS) {
|
||||
// Fill the gap.
|
||||
for(; hoffset < offset[sh] ; hoffset += 4) {
|
||||
ef->data[dpos++] = 0;
|
||||
}
|
||||
// Copy the data.
|
||||
for (int spos = hunks[sh].datastart ; spos < hunks[sh].datastart + hunks[sh].datasize ; spos++) {
|
||||
ef->data[dpos++] = data[spos];
|
||||
}
|
||||
hoffset += hunks[sh].datasize*4;
|
||||
}
|
||||
}
|
||||
|
||||
// Transfer all reloc information to the new hunk.
|
||||
ef->data[dpos++] = HUNK_RELOC32;
|
||||
ef->hunks[dh].relocstart = dpos;
|
||||
ef->hunks[dh].relocentries = 0;
|
||||
unsigned char *bytes = (unsigned char *)&ef->data[ef->hunks[dh].datastart];
|
||||
// Iterate through destination reloc target hunk
|
||||
for (int drh = 0 ; drh < ef->hunks.size() ; drh++) {
|
||||
// Make space for number of relocs and store index of target hunk.
|
||||
int rnpos = dpos++; // Position for number of relocs
|
||||
ef->data[dpos++] = drh;
|
||||
// Transfer all appropriate reloc entries.
|
||||
int rtot = 0; // Total number of relocs in hunk
|
||||
for (int sh = 0 ; sh < numhunks ; sh++) {
|
||||
if (dhunk[sh] == dh && hunks[sh].relocentries > 0) {
|
||||
int spos = hunks[sh].relocstart;
|
||||
int rn; // Number of relocs
|
||||
while ((rn = data[spos++]) > 0) {
|
||||
int srh = data[spos++]; // Source reloc target hunk
|
||||
if (dhunk[srh] == drh) {
|
||||
rtot += rn;
|
||||
for (int ri = 0 ; ri < rn ; ri++) {
|
||||
int rv = data[spos++]; // Reloc value
|
||||
ef->data[dpos++] = rv+offset[sh];
|
||||
*((Longword *)&bytes[rv+offset[sh]]) += offset[srh];
|
||||
}
|
||||
} else {
|
||||
spos += rn;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Store total number of relocs with the actual target hunk.
|
||||
// If there are none, remove the spaces for
|
||||
// number of relocs and target hunk.
|
||||
if (rtot == 0) {
|
||||
dpos -= 2;
|
||||
} else {
|
||||
ef->data[rnpos] = rtot;
|
||||
ef->hunks[dh].relocentries += rtot;
|
||||
}
|
||||
}
|
||||
// End the reloc section.
|
||||
// If there are no relocs, remove the reloc header.
|
||||
if (ef->hunks[dh].relocentries == 0) {
|
||||
dpos -= 1;
|
||||
} else {
|
||||
ef->data[dpos++] = 0;
|
||||
}
|
||||
}
|
||||
// There must be a HUNK_END after last hunk!
|
||||
ef->data[dpos++] = HUNK_END;
|
||||
// Note resulting file size
|
||||
ef->data.resize(dpos);
|
||||
|
||||
return ef;
|
||||
}
|
||||
|
||||
bool valid_mini() {
|
||||
if (!(hunks[0].type == HUNK_CODE && hunks[0].relocentries == 0)) return false;
|
||||
for (int h = 1 ; h < hunks.size() ; h++) {
|
||||
if (hunks[h].relocentries != 0) return false;
|
||||
if (hunks[h].type == HUNK_BSS || hunks[h].datasize == 0) continue;
|
||||
for (int i = 0 ; i < hunks[h].datasize ; i++) {
|
||||
if (data[hunks[h].datastart + i] != 0) return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
HunkFile* crunch(PackParams *params, bool overlap, bool mini, string *decrunch_text, unsigned flash_address, RefEdgeFactory *edge_factory, bool show_progress) {
|
||||
vector<unsigned> pack_buffer = compress_hunks(params, overlap, mini, edge_factory, show_progress);
|
||||
vector<pair<int,int> > count_and_hunksize = verify(pack_buffer, overlap, mini);
|
||||
|
||||
int numhunks = hunks.size();
|
||||
int newnumhunks = numhunks+1;
|
||||
int bufsize = data.size() * 11 / 10 + 1000;
|
||||
|
||||
HunkFile *ef = new HunkFile;
|
||||
ef->data.resize(bufsize, 0);
|
||||
|
||||
int dpos = 0;
|
||||
|
||||
// Write new hunk header
|
||||
ef->data[dpos++] = HUNK_HEADER;
|
||||
ef->data[dpos++] = 0;
|
||||
ef->data[dpos++] = newnumhunks;
|
||||
ef->data[dpos++] = 0;
|
||||
ef->data[dpos++] = newnumhunks-1;
|
||||
|
||||
int lpos1, lpos2, ppos;
|
||||
Word *offsetp = NULL;
|
||||
if (overlap) {
|
||||
// Write hunk memory sizes
|
||||
lpos1 = dpos++;
|
||||
for (int h = 0 ; h < numhunks ; h++) {
|
||||
int hunksize = max(hunks[h].memsize, count_and_hunksize[h].second);
|
||||
ef->data[dpos++] = hunksize | hunks[h].flags;
|
||||
}
|
||||
|
||||
// Write header
|
||||
ef->data[dpos++] = HUNK_CODE;
|
||||
lpos2 = dpos++;
|
||||
ppos = dpos;
|
||||
if (decrunch_text) {
|
||||
memcpy(&ef->data[dpos], OverlapHeaderT, sizeof(OverlapHeaderT));
|
||||
dpos += sizeof(OverlapHeaderT) / sizeof(Longword);
|
||||
ef->data[ppos + 4] = decrunch_text->length();
|
||||
offsetp = (Word *) &ef->data[ppos + 10];
|
||||
} else {
|
||||
memcpy(&ef->data[dpos], OverlapHeader, sizeof(OverlapHeader));
|
||||
dpos += sizeof(OverlapHeader) / sizeof(Longword);
|
||||
}
|
||||
} else if (mini) {
|
||||
// Write hunk memory sizes
|
||||
lpos1 = dpos++;
|
||||
for (int h = 0 ; h < numhunks ; h++) {
|
||||
ef->data[dpos++] = hunks[h].memsize | hunks[h].flags;
|
||||
}
|
||||
|
||||
// Write header
|
||||
ef->data[dpos++] = HUNK_CODE;
|
||||
lpos2 = dpos++;
|
||||
ppos = dpos;
|
||||
memcpy(&ef->data[dpos], MiniHeader, sizeof(MiniHeader));
|
||||
dpos += sizeof(MiniHeader) / sizeof(Longword);
|
||||
offsetp = (Word *) (((unsigned char *) &ef->data[ppos]) + 12);
|
||||
} else {
|
||||
int header1_size = sizeof(Header1) / sizeof(Longword);
|
||||
if (decrunch_text) {
|
||||
header1_size = (sizeof(Header1T) + (decrunch_text->length() + 3)) / sizeof(Longword);
|
||||
}
|
||||
for (int h = 0 ; h < numhunks ; h++) {
|
||||
int memsize = hunks[h].memsize;
|
||||
if (h == 0 && memsize < header1_size) {
|
||||
// Make space for header trampoline code
|
||||
memsize = header1_size;
|
||||
}
|
||||
ef->data[dpos++] = memsize | hunks[h].flags;
|
||||
}
|
||||
lpos1 = dpos++;
|
||||
|
||||
// Write header 1
|
||||
ef->data[dpos++] = HUNK_CODE;
|
||||
ef->data[dpos++] = header1_size;
|
||||
if (decrunch_text) {
|
||||
memset(&ef->data[dpos], 0, header1_size);
|
||||
memcpy(&ef->data[dpos], Header1T, sizeof(Header1T));
|
||||
char *text_dest = ((char *) &ef->data[dpos]) + sizeof(Header1T);
|
||||
memcpy(text_dest, decrunch_text->c_str(), decrunch_text->length());
|
||||
ef->data[dpos + 5] = decrunch_text->length();
|
||||
} else {
|
||||
memcpy(&ef->data[dpos], Header1, sizeof(Header1));
|
||||
}
|
||||
dpos += header1_size;
|
||||
|
||||
// Write hunks
|
||||
for (int h = 1 ; h < numhunks ; h++) {
|
||||
ef->data[dpos++] = hunks[h].type;
|
||||
switch (hunks[h].type) {
|
||||
case HUNK_CODE:
|
||||
case HUNK_DATA:
|
||||
ef->data[dpos++] = 0;
|
||||
break;
|
||||
case HUNK_BSS:
|
||||
ef->data[dpos++] = hunks[h].datasize;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Write header 2
|
||||
ef->data[dpos++] = HUNK_CODE;
|
||||
lpos2 = dpos++;
|
||||
ppos = dpos;
|
||||
memcpy(&ef->data[dpos], Header2, sizeof(Header2));
|
||||
dpos += sizeof(Header2) / sizeof(Longword);
|
||||
offsetp = (Word *) (((unsigned char *) &ef->data[ppos]) + 4);
|
||||
}
|
||||
|
||||
if (flash_address) {
|
||||
// Insert flashing code
|
||||
dpos += 1;
|
||||
for (int fpos = dpos - 1 ; fpos >= dpos - 9 ; fpos--) {
|
||||
ef->data[fpos] = ef->data[fpos - 1];
|
||||
}
|
||||
Word* insts = (Word *) &ef->data[dpos - 11];
|
||||
insts[0] = 0x33C3; // move.w d3,flash_address
|
||||
*(Longword *)&insts[1] = flash_address;
|
||||
insts[3] = 0x6AEC; // bpl.b readbit
|
||||
if (offsetp) *offsetp += 4;
|
||||
}
|
||||
|
||||
if (overlap) {
|
||||
// Write decrunch text
|
||||
if (decrunch_text) {
|
||||
int rounded_text_size = (decrunch_text->length() + 3) & -4;
|
||||
memset(&ef->data[dpos], 0, rounded_text_size);
|
||||
memcpy(&ef->data[dpos], decrunch_text->c_str(), decrunch_text->length());
|
||||
dpos += rounded_text_size / sizeof(Longword);
|
||||
}
|
||||
|
||||
// Set hunk sizes
|
||||
ef->data[lpos1] = dpos-ppos;
|
||||
ef->data[lpos2] = dpos-ppos;
|
||||
|
||||
// Write hunks
|
||||
int packed_index = 0;
|
||||
for (int h = 0 ; h < numhunks ; h++) {
|
||||
ef->data[dpos++] = HUNK_DATA;
|
||||
int longwords_in_hunk = min<int>(count_and_hunksize[h].first, pack_buffer.size() - packed_index);
|
||||
ef->data[dpos++] = longwords_in_hunk + 1;
|
||||
ef->data[dpos++] = count_and_hunksize[h].first * 4;
|
||||
for (int i = 0 ; i < longwords_in_hunk ; i++) {
|
||||
ef->data[dpos++] = pack_buffer[packed_index++];
|
||||
}
|
||||
}
|
||||
} else if (mini) {
|
||||
// Write compressed data backwards
|
||||
for (int i = pack_buffer.size()-1 ; i >= 0 ; i--) {
|
||||
ef->data[dpos++] = pack_buffer[i];
|
||||
}
|
||||
|
||||
// Set hunk sizes
|
||||
ef->data[lpos1] = dpos-ppos + 32768/8*2/4; // Space for context state
|
||||
ef->data[lpos2] = dpos-ppos;
|
||||
|
||||
// Write hunks
|
||||
for (int h = 0 ; h < numhunks ; h++) {
|
||||
ef->data[dpos++] = HUNK_BSS;
|
||||
ef->data[dpos++] = hunks[h].memsize;
|
||||
}
|
||||
|
||||
// Set size of data in header
|
||||
int offset = (int) *offsetp + pack_buffer.size() * 4;
|
||||
if (offset > 32767) {
|
||||
printf("Size overflow: final size in mini mode must be less than 24k.\n\n");
|
||||
exit(1);
|
||||
}
|
||||
*offsetp = offset;
|
||||
} else {
|
||||
// Write compressed data
|
||||
for (int i = 0 ; i < pack_buffer.size() ; i++) {
|
||||
ef->data[dpos++] = pack_buffer[i];
|
||||
}
|
||||
|
||||
// Set hunk sizes
|
||||
ef->data[lpos1] = dpos-ppos + 1; // Space for range decoder overshoot
|
||||
ef->data[lpos2] = dpos-ppos;
|
||||
}
|
||||
|
||||
// There must be a HUNK_END after last hunk!
|
||||
ef->data[dpos++] = HUNK_END;
|
||||
// Note resulting file size
|
||||
ef->data.resize(dpos);
|
||||
|
||||
return ef;
|
||||
}
|
||||
};
|
||||
Vendored
+35
@@ -0,0 +1,35 @@
|
||||
|
||||
Shrinkler executable file compressor for Amiga
|
||||
|
||||
Copyright 1999-2015 Aske Simon Christensen, with exceptions noted below.
|
||||
|
||||
Permission is hereby granted to anyone obtaining a copy of this software
|
||||
package (including accompanying documentation) to compile, use, copy,
|
||||
modify, merge and/or distribute it, in whole or in part, subject to the
|
||||
following conditions:
|
||||
|
||||
- Distribution in source code form must include a copy of this license.
|
||||
|
||||
- Distribution in binary form must not be misattributed, i.e. you must
|
||||
not claim (implicitly or explicitly) that you wrote it yourself.
|
||||
|
||||
- Distribution of the decrunch headers (Header.S, MiniHeader.S,
|
||||
OverlapHeader.S, and the .bin and .dat files generated from them) in
|
||||
binary form as part of an Amiga executable is not restricted by this
|
||||
license and does not require attribution.
|
||||
In particular, output executables from Shrinkler (which contain code
|
||||
from the decrunch headers) are to be considered original works of the
|
||||
author(s) of the corresponding input executables.
|
||||
|
||||
- The data decompression code (ShrinklerDecompress.S) is distributed
|
||||
alongside the Shrinkler binaries in the official archives and has its
|
||||
own license stated inside the file.
|
||||
|
||||
Exceptions:
|
||||
|
||||
- doshunks.h is part of the Amiga SDK and is Copyright 1989-1993
|
||||
Commodore-Amiga, Inc.
|
||||
|
||||
- The "as" and "ld" executables in the toolchain directory are taken
|
||||
from the AmiDevCpp distribution. They are covered by the GNU General
|
||||
Public License.
|
||||
Vendored
+74
@@ -0,0 +1,74 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
Decoder for the LZ encoder.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Decoder.h"
|
||||
#include "LZEncoder.h"
|
||||
|
||||
class LZReceiver {
|
||||
public:
|
||||
virtual bool receiveLiteral(unsigned char value) = 0;
|
||||
virtual bool receiveReference(int offset, int length) = 0;
|
||||
virtual ~LZReceiver() {}
|
||||
};
|
||||
|
||||
class LZDecoder {
|
||||
Decoder *decoder;
|
||||
|
||||
int decode(int context) const {
|
||||
return decoder->decode(LZEncoder::NUM_SINGLE_CONTEXTS + context);
|
||||
}
|
||||
|
||||
int decodeNumber(int context_group) const {
|
||||
return decoder->decodeNumber(LZEncoder::NUM_SINGLE_CONTEXTS + (context_group << 8));
|
||||
}
|
||||
|
||||
public:
|
||||
LZDecoder(Decoder *decoder) : decoder(decoder) {
|
||||
|
||||
}
|
||||
|
||||
bool decode(LZReceiver& receiver) {
|
||||
bool ref = false;
|
||||
bool prev_was_ref = false;
|
||||
int pos = 0;
|
||||
int offset = 0;
|
||||
do {
|
||||
if (ref) {
|
||||
bool repeated = false;
|
||||
if (!prev_was_ref) {
|
||||
repeated = decode(LZEncoder::CONTEXT_REPEATED);
|
||||
}
|
||||
if (!repeated) {
|
||||
offset = decodeNumber(LZEncoder::CONTEXT_GROUP_OFFSET) - 2;
|
||||
if (offset == 0) break;
|
||||
}
|
||||
int length = decodeNumber(LZEncoder::CONTEXT_GROUP_LENGTH);
|
||||
if (!receiver.receiveReference(offset, length)) return false;
|
||||
pos += length;
|
||||
prev_was_ref = true;
|
||||
} else {
|
||||
int parity = pos & 1;
|
||||
int context = 1;
|
||||
for (int i = 7 ; i >= 0 ; i--) {
|
||||
int bit = decode((parity << 8) | context);
|
||||
context = (context << 1) | bit;
|
||||
}
|
||||
unsigned char lit = context;
|
||||
if (!receiver.receiveLiteral(lit)) return false;
|
||||
pos += 1;
|
||||
prev_was_ref = false;
|
||||
}
|
||||
int parity = pos & 1;
|
||||
ref = decode(LZEncoder::CONTEXT_KIND + (parity << 8));
|
||||
} while (true);
|
||||
return true;
|
||||
}
|
||||
|
||||
};
|
||||
Vendored
+182
@@ -0,0 +1,182 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
The LZ encoder defines the encoding of LZ symbols (literal bytes and references) into data bytes.
|
||||
|
||||
The encoding consists of three layers:
|
||||
|
||||
Layer 1 defines a plain encoding into bits. It is as follows:
|
||||
|
||||
The first synbol is always a literal, and it is encoded as
|
||||
|
||||
bit7 .. bit0
|
||||
|
||||
Subsequent symbols can be either literals or references, and are encoded as one of
|
||||
|
||||
0 bit7 .. bit0 (literal byte)
|
||||
1 0 <offset+2> <length> (reference)
|
||||
1 1 <length> (reference with same offset as previous reference)
|
||||
|
||||
and the data block for each hunk is terminated by
|
||||
|
||||
1 0 <2>
|
||||
|
||||
The data block is followed by relocation entries, specifying positions within the data where the
|
||||
address of some hunk must be added. The entries are separated into one list for each hunk.
|
||||
Within each list, each entry is encoded as
|
||||
|
||||
<delta from previous position> (the position before the first entry is assumed to be -4)
|
||||
|
||||
and each list is terminated by
|
||||
|
||||
<2>
|
||||
|
||||
The <number> encodings in the above are variable-length numbers with a value of 2 or greater.
|
||||
The number 1 bit(n-1) .. bit0 is encoded as
|
||||
|
||||
1^(n-1) 0 bit(n-1) .. bit0
|
||||
|
||||
Layer 2 defines a context for each bit of the Layer 1 encoding. The probability distribution
|
||||
between 0 and 1 is modelled adaptively for each context.
|
||||
|
||||
The first bit of the general symbol encoding (the one that selects between literal and reference)
|
||||
has one context for each parity of the byte position in the data (i.e. one for even bytes and one
|
||||
for odd bytes).
|
||||
|
||||
The second bit of the reference symbol encoding (the one that selects between new and repeated
|
||||
offset) has a single context for itself.
|
||||
|
||||
Literal bits have one context for each combination of parity and all higher numbered bits within
|
||||
the same literal byte. Thus, there are 510 different literal contexts.
|
||||
|
||||
Numbers have one context group for each of offset, length and relocation entry. Within each group,
|
||||
there is one context for each of the prefix bits, and one context for each data bit number (i.e.
|
||||
bit(i) always uses the same context for all numbers with more than i data bits).
|
||||
|
||||
Layer 3 performs entropy coding of the Layer 1 bits based on the probabilities estimated by Layer 2.
|
||||
The entropy coder defines the final compressed data contents.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Coder.h"
|
||||
|
||||
class LZState {
|
||||
unsigned after_first:1;
|
||||
unsigned prev_was_ref:1;
|
||||
unsigned parity:1;
|
||||
unsigned last_offset:28;
|
||||
|
||||
friend class LZEncoder;
|
||||
};
|
||||
|
||||
class LZEncoder {
|
||||
static const int NUM_SINGLE_CONTEXTS = 1;
|
||||
static const int NUM_CONTEXT_GROUPS = 4;
|
||||
static const int CONTEXT_GROUP_SIZE = 256;
|
||||
|
||||
static const int CONTEXT_KIND = 0;
|
||||
static const int CONTEXT_REPEATED = -1;
|
||||
|
||||
static const int CONTEXT_GROUP_LIT = 0;
|
||||
static const int CONTEXT_GROUP_OFFSET = 2;
|
||||
static const int CONTEXT_GROUP_LENGTH = 3;
|
||||
|
||||
Coder *coder;
|
||||
|
||||
int code(int context, int bit) const {
|
||||
return coder->code(NUM_SINGLE_CONTEXTS + context, bit);
|
||||
}
|
||||
|
||||
int encodeNumber(int context_group, int number) const {
|
||||
return coder->encodeNumber(NUM_SINGLE_CONTEXTS + (context_group << 8), number);
|
||||
}
|
||||
|
||||
friend class LZDecoder;
|
||||
|
||||
public:
|
||||
static const int KIND_LIT = 0;
|
||||
static const int KIND_REF = 1;
|
||||
|
||||
static const int NUM_CONTEXTS = (NUM_SINGLE_CONTEXTS + NUM_CONTEXT_GROUPS * CONTEXT_GROUP_SIZE);
|
||||
static const int NUMBER_CONTEXT_OFFSET = (NUM_SINGLE_CONTEXTS + CONTEXT_GROUP_OFFSET * CONTEXT_GROUP_SIZE);
|
||||
static const int NUM_NUMBER_CONTEXTS = 2;
|
||||
|
||||
LZEncoder(Coder *coder) : coder(coder) {
|
||||
|
||||
}
|
||||
|
||||
void setInitialState(LZState *state) const {
|
||||
state->after_first = 0;
|
||||
state->prev_was_ref = 0;
|
||||
state->parity = 0;
|
||||
state->last_offset = 0;
|
||||
}
|
||||
|
||||
void constructState(LZState *state, int pos, bool prev_was_ref, int last_offset) const {
|
||||
state->after_first = pos > 0;
|
||||
state->prev_was_ref = prev_was_ref;
|
||||
state->parity = pos;
|
||||
state->last_offset = last_offset;
|
||||
}
|
||||
|
||||
int encodeLiteral(unsigned char value, const LZState *state_before, LZState *state_after) const {
|
||||
int size = 0;
|
||||
if (state_before->after_first) {
|
||||
size += code(CONTEXT_KIND + (state_before->parity << 8), KIND_LIT);
|
||||
}
|
||||
int context = 1;
|
||||
for (int i = 7 ; i >= 0 ; i--) {
|
||||
int bit = ((value >> i) & 1);
|
||||
size += code((state_before->parity << 8) | context, bit);
|
||||
context = (context << 1) | bit;
|
||||
}
|
||||
|
||||
state_after->after_first = 1;
|
||||
state_after->prev_was_ref = 0;
|
||||
state_after->parity = state_before->parity + 1;
|
||||
state_after->last_offset = state_before->last_offset;
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
int encodeReference(int offset, int length, const LZState *state_before, LZState *state_after) const {
|
||||
assert(offset >= 1);
|
||||
assert(length >= 2);
|
||||
assert(state_before->after_first);
|
||||
|
||||
int size = code(CONTEXT_KIND + (state_before->parity << 8), KIND_REF);
|
||||
int rep_offset = offset == state_before->last_offset;
|
||||
if (!state_before->prev_was_ref) {
|
||||
size += code(CONTEXT_REPEATED, rep_offset);
|
||||
} else {
|
||||
assert(!rep_offset);
|
||||
}
|
||||
if (!rep_offset) {
|
||||
size += encodeNumber(CONTEXT_GROUP_OFFSET, offset + 2);
|
||||
}
|
||||
size += encodeNumber(CONTEXT_GROUP_LENGTH, length);
|
||||
|
||||
state_after->after_first = 1;
|
||||
state_after->prev_was_ref = 1;
|
||||
state_after->parity = state_before->parity + length;
|
||||
state_after->last_offset = offset;
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
int finish(const LZState *state_before) const {
|
||||
int size = code(CONTEXT_KIND + (state_before->parity << 8), KIND_REF);
|
||||
if (!state_before->prev_was_ref) {
|
||||
size += code(CONTEXT_REPEATED, 0);
|
||||
}
|
||||
int context_group = CONTEXT_GROUP_OFFSET;
|
||||
int number = 2;
|
||||
size += encodeNumber(context_group, number);
|
||||
|
||||
return size;
|
||||
}
|
||||
};
|
||||
|
||||
Vendored
+408
@@ -0,0 +1,408 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
Parse a data block into LZ symbols (literal bytes and references).
|
||||
|
||||
The parser uses a "local optimal parse" strategy, where all matches reported
|
||||
by the match finder are considered. Potential parses are maintained for each
|
||||
possible previous reference offset, in order to maximize the utilization of
|
||||
the "repeated offset" feature of the LZ encoding.
|
||||
|
||||
Three parameters control the speed/precision tradeoff of the parser:
|
||||
|
||||
The length_margin parameter how many shorter matches the parser will consider
|
||||
for each match reported by the match finder. If the match finder reports a
|
||||
match of length l, the parser will consider all (valid) matches of length at
|
||||
least l-length_margin.
|
||||
|
||||
The skip_length parameter controls a shortcutting mechanism for very long
|
||||
matches. Whenever a match of length at least skip_length is reported, the
|
||||
parser will use that match unconditionally and skip ahead to continue the
|
||||
parsing at the end of the match.
|
||||
|
||||
The max_edges parameter controls the total number of reference edges the
|
||||
parser will keep around for representing potential parses. Whenever the
|
||||
limit is reached, the parser will delete the least favorable of the current
|
||||
parses to free up space.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <functional>
|
||||
#include <utility>
|
||||
#include <list>
|
||||
#include <algorithm>
|
||||
|
||||
using std::map;
|
||||
using std::max;
|
||||
using std::min;
|
||||
using std::pair;
|
||||
using std::sort;
|
||||
using std::vector;
|
||||
|
||||
#include "LZEncoder.h"
|
||||
#include "MatchFinder.h"
|
||||
#include "Heap.h"
|
||||
#include "CuckooHash.h"
|
||||
#include "assert.h"
|
||||
|
||||
// For each offset:
|
||||
// Best total size with last ref having that offset
|
||||
|
||||
class RefEdge {
|
||||
int pos;
|
||||
int offset;
|
||||
int length;
|
||||
int total_size;
|
||||
int refcount;
|
||||
RefEdge *source;
|
||||
|
||||
RefEdge(int pos, int offset, int length, int total_size, RefEdge *source)
|
||||
: pos(pos), offset(offset), length(length), total_size(total_size), source(source)
|
||||
{
|
||||
assert(source != this);
|
||||
refcount = 1;
|
||||
if (source != NULL) {
|
||||
source->refcount++;
|
||||
}
|
||||
}
|
||||
|
||||
int target() {
|
||||
return pos + length;
|
||||
}
|
||||
|
||||
friend class RefEdgeFactory;
|
||||
friend class LZParser;
|
||||
friend class LZResultEdge;
|
||||
friend class LZParseResult;
|
||||
friend struct std::less<RefEdge*>;
|
||||
|
||||
public:
|
||||
int _heap_index;
|
||||
};
|
||||
|
||||
namespace std {
|
||||
template <> struct less<RefEdge*> {
|
||||
bool operator()(RefEdge* const & e1, RefEdge* const & e2) const {
|
||||
return e1->total_size < e2->total_size;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// Factory for RefEdge objects which recycles destroyed objects for efficiency
|
||||
class RefEdgeFactory {
|
||||
int edge_capacity;
|
||||
int edge_count;
|
||||
int cleaned_edges;
|
||||
|
||||
RefEdge* buffer;
|
||||
public:
|
||||
int max_edge_count;
|
||||
int max_cleaned_edges;
|
||||
|
||||
RefEdgeFactory(int edge_capacity) : edge_capacity(edge_capacity),
|
||||
edge_count(0), cleaned_edges(0), max_edge_count(0), max_cleaned_edges(0)
|
||||
{
|
||||
buffer = NULL;
|
||||
}
|
||||
|
||||
~RefEdgeFactory() {
|
||||
while (buffer != NULL) {
|
||||
RefEdge *edge = buffer;
|
||||
buffer = buffer->source;
|
||||
delete edge;
|
||||
}
|
||||
}
|
||||
|
||||
void reset() {
|
||||
assert(edge_count == 0);
|
||||
cleaned_edges = 0;
|
||||
}
|
||||
|
||||
RefEdge* create(int pos, int offset, int length, int total_size, RefEdge *source) {
|
||||
max_edge_count = max(max_edge_count, ++edge_count);
|
||||
if (buffer == NULL) {
|
||||
return new RefEdge(pos, offset, length, total_size, source);
|
||||
} else {
|
||||
RefEdge* edge = buffer;
|
||||
buffer = edge->source;
|
||||
return new (edge) RefEdge(pos, offset, length, total_size, source);
|
||||
}
|
||||
}
|
||||
|
||||
void destroy(RefEdge* edge, bool clean) {
|
||||
edge->source = buffer;
|
||||
buffer = edge;
|
||||
edge_count--;
|
||||
if (clean) {
|
||||
max_cleaned_edges = max(max_cleaned_edges, ++cleaned_edges);
|
||||
}
|
||||
}
|
||||
|
||||
bool full() {
|
||||
return edge_count >= edge_capacity;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
class LZProgress {
|
||||
public:
|
||||
virtual void begin(int size) = 0;
|
||||
virtual void update(int pos) = 0;
|
||||
virtual void end() = 0;
|
||||
|
||||
virtual ~LZProgress() {}
|
||||
};
|
||||
|
||||
struct LZResultEdge {
|
||||
int pos;
|
||||
int offset;
|
||||
int length;
|
||||
|
||||
LZResultEdge(RefEdge *edge) : pos(edge->pos), offset(edge->offset), length(edge->length) {}
|
||||
|
||||
friend class LZParseResult;
|
||||
};
|
||||
|
||||
class LZParseResult {
|
||||
vector<LZResultEdge> edges;
|
||||
const unsigned char *data;
|
||||
int data_length;
|
||||
int zero_padding;
|
||||
public:
|
||||
int encode(const LZEncoder& result_encoder) const {
|
||||
int size = 0;
|
||||
int pos = 0;
|
||||
LZState state;
|
||||
result_encoder.setInitialState(&state);
|
||||
for (int i = edges.size() - 1 ; i >= 0 ; i--) {
|
||||
const LZResultEdge *edge = &edges[i];
|
||||
while (pos < edge->pos) {
|
||||
size += result_encoder.encodeLiteral(data[pos++], &state, &state);
|
||||
}
|
||||
size += result_encoder.encodeReference(edge->offset, edge->length, &state, &state);
|
||||
pos += edge->length;
|
||||
}
|
||||
while (pos < data_length) {
|
||||
size += result_encoder.encodeLiteral(data[pos++], &state, &state);
|
||||
}
|
||||
if (zero_padding > 0) {
|
||||
size += result_encoder.encodeLiteral(0, &state, &state);
|
||||
if (zero_padding == 2) {
|
||||
size += result_encoder.encodeLiteral(0, &state, &state);
|
||||
} else if (zero_padding > 1) {
|
||||
size += result_encoder.encodeReference(1, zero_padding - 1, &state, &state);
|
||||
}
|
||||
}
|
||||
size += result_encoder.finish(&state);
|
||||
return size;
|
||||
}
|
||||
|
||||
friend class LZParser;
|
||||
};
|
||||
|
||||
class LZParser {
|
||||
const unsigned char *data;
|
||||
int data_length;
|
||||
int zero_padding;
|
||||
MatchFinder& finder;
|
||||
int length_margin;
|
||||
int skip_length;
|
||||
const LZEncoder* encoderp;
|
||||
RefEdgeFactory* edge_factory;
|
||||
|
||||
vector<int> literal_size;
|
||||
vector<CuckooHash<RefEdge*> > edges_to_pos;
|
||||
RefEdge* best;
|
||||
CuckooHash<RefEdge*> best_for_offset;
|
||||
Heap<RefEdge*> root_edges;
|
||||
|
||||
bool is_root(RefEdge *edge) {
|
||||
return root_edges.contains(edge);
|
||||
}
|
||||
|
||||
void remove_root(RefEdge *edge) {
|
||||
root_edges.remove(edge);
|
||||
}
|
||||
|
||||
void releaseEdge(RefEdge *edge, bool clean = false) {
|
||||
while (edge != NULL) {
|
||||
RefEdge *source = edge->source;
|
||||
if (--edge->refcount == 0) {
|
||||
assert(!is_root(edge));
|
||||
edge_factory->destroy(edge, clean);
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
edge = source;
|
||||
}
|
||||
}
|
||||
|
||||
// Return progress
|
||||
bool clean_worst_edge(int pos, RefEdge *exclude) {
|
||||
if (root_edges.size() == 0) return false;
|
||||
RefEdge *worst_edge = root_edges.remove_largest();
|
||||
if (worst_edge == best || worst_edge == exclude) return true;
|
||||
CuckooHash<RefEdge*>& container = worst_edge->target() > pos
|
||||
? edges_to_pos[worst_edge->target()]
|
||||
: best_for_offset;
|
||||
if (container.size() > 1 && container.count(worst_edge->offset) > 0) {
|
||||
container.erase(worst_edge->offset);
|
||||
releaseEdge(worst_edge, true);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void put_by_offset(CuckooHash<RefEdge*>& by_offset, RefEdge* edge) {
|
||||
assert(!is_root(edge));
|
||||
if (by_offset.count(edge->offset) == 0) {
|
||||
by_offset[edge->offset] = edge;
|
||||
root_edges.insert(edge);
|
||||
} else if (edge->total_size < by_offset[edge->offset]->total_size) {
|
||||
RefEdge* old_edge = by_offset[edge->offset];
|
||||
remove_root(old_edge);
|
||||
releaseEdge(old_edge);
|
||||
by_offset[edge->offset] = edge;
|
||||
root_edges.insert(edge);
|
||||
} else {
|
||||
releaseEdge(edge);
|
||||
}
|
||||
}
|
||||
|
||||
void newEdge(RefEdge *source, int pos, int offset, int length) {
|
||||
if (source && offset == source->offset && pos == source->target()) return;
|
||||
int prev_target = source ? source->target() : 0;
|
||||
int new_target = pos + length;
|
||||
LZState state_before;
|
||||
LZState state_after;
|
||||
encoderp->constructState(&state_before, pos, pos == prev_target, source ? source->offset : 0);
|
||||
int size_before = (source ? source->total_size : literal_size[data_length]) - (literal_size[data_length] - literal_size[pos]);
|
||||
int edge_size = encoderp->encodeReference(offset, length, &state_before, &state_after);
|
||||
int size_after = literal_size[data_length] - literal_size[new_target];
|
||||
while (edge_factory->full()) {
|
||||
if (!clean_worst_edge(pos, source)) break;
|
||||
}
|
||||
RefEdge *new_edge = edge_factory->create(pos, offset, length, size_before + edge_size + size_after, source);
|
||||
put_by_offset(edges_to_pos[new_target], new_edge);
|
||||
}
|
||||
|
||||
public:
|
||||
LZParser(const unsigned char *data, int data_length, int zero_padding, MatchFinder& finder, int length_margin, int skip_length, RefEdgeFactory* edge_factory)
|
||||
: data(data), data_length(data_length), zero_padding(zero_padding), finder(finder), length_margin(length_margin), skip_length(skip_length), edge_factory(edge_factory)
|
||||
{
|
||||
// Initialize edges_to_pos array
|
||||
edges_to_pos.resize(data_length + 1);
|
||||
best = NULL;
|
||||
}
|
||||
|
||||
LZParseResult parse(const LZEncoder& encoder, LZProgress *progress) {
|
||||
progress->begin(data_length);
|
||||
encoderp = &encoder;
|
||||
|
||||
// Reset state
|
||||
best_for_offset.clear();
|
||||
root_edges.clear();
|
||||
edge_factory->reset();
|
||||
|
||||
// Accumulate literal sizes
|
||||
literal_size.resize(data_length + 1, 0);
|
||||
int size = 0;
|
||||
LZState literal_state;
|
||||
encoder.setInitialState(&literal_state);
|
||||
for (int i = 0 ; i < data_length ; i++) {
|
||||
literal_size[i] = size;
|
||||
size += encoder.encodeLiteral(data[i], &literal_state, &literal_state);
|
||||
}
|
||||
literal_size[data_length] = size;
|
||||
|
||||
// Parse
|
||||
RefEdge* initial_best = edge_factory->create(0, 0, 0, literal_size[data_length], NULL);
|
||||
best = initial_best;
|
||||
for (int pos = 1 ; pos <= data_length ; pos++) {
|
||||
// Assimilate edges ending here
|
||||
for (CuckooHash<RefEdge*>::iterator it = edges_to_pos[pos].begin() ; it != edges_to_pos[pos].end() ; it++) {
|
||||
RefEdge *edge = it->second;
|
||||
if (edge->total_size < best->total_size) {
|
||||
best = edge;
|
||||
}
|
||||
remove_root(edge);
|
||||
put_by_offset(best_for_offset, edge);
|
||||
}
|
||||
edges_to_pos[pos].clear();
|
||||
|
||||
// Add new edges according to matches
|
||||
finder.beginMatching(pos);
|
||||
int match_pos;
|
||||
int match_length;
|
||||
int max_match_length = 0;
|
||||
while (finder.nextMatch(&match_pos, &match_length)) {
|
||||
int offset = pos - match_pos;
|
||||
if (match_length > data_length - pos) {
|
||||
match_length = data_length - pos;
|
||||
}
|
||||
int min_length = match_length - length_margin;
|
||||
if (min_length < 2) min_length = 2;
|
||||
for (int length = min_length ; length <= match_length ; length++) {
|
||||
newEdge(best, pos, offset, length);
|
||||
if (best->offset != offset && best_for_offset.count(offset)) {
|
||||
assert(best_for_offset[offset]->target() <= pos);
|
||||
newEdge(best_for_offset[offset], pos, offset, length);
|
||||
}
|
||||
}
|
||||
max_match_length = max(max_match_length, match_length);
|
||||
}
|
||||
|
||||
// If we have a very long match, skip ahead
|
||||
if (max_match_length >= skip_length && !edges_to_pos[pos + max_match_length].empty()) {
|
||||
root_edges.clear();
|
||||
for (CuckooHash<RefEdge*>::iterator it = best_for_offset.begin() ; it != best_for_offset.end() ; it++) {
|
||||
releaseEdge(it->second);
|
||||
}
|
||||
best_for_offset.clear();
|
||||
int target_pos = pos + max_match_length;
|
||||
while (pos < target_pos - 1) {
|
||||
CuckooHash<RefEdge*>& edges = edges_to_pos[++pos];
|
||||
for (CuckooHash<RefEdge*>::iterator it = edges.begin() ; it != edges.end() ; it++) {
|
||||
releaseEdge(it->second);
|
||||
}
|
||||
edges.clear();
|
||||
}
|
||||
best = initial_best;
|
||||
}
|
||||
|
||||
progress->update(pos);
|
||||
}
|
||||
|
||||
// Clean unused paths
|
||||
root_edges.clear();
|
||||
for (CuckooHash<RefEdge*>::iterator it = best_for_offset.begin() ; it != best_for_offset.end() ; it++) {
|
||||
RefEdge *edge = it->second;
|
||||
if (edge != best) {
|
||||
releaseEdge(edge);
|
||||
}
|
||||
}
|
||||
|
||||
// Find best path
|
||||
LZParseResult result;
|
||||
result.data = data;
|
||||
result.data_length = data_length;
|
||||
result.zero_padding = zero_padding;
|
||||
RefEdge *edge = best;
|
||||
while (edge->length > 0) {
|
||||
result.edges.push_back(LZResultEdge(edge));
|
||||
edge = edge->source;
|
||||
}
|
||||
releaseEdge(edge);
|
||||
releaseEdge(best);
|
||||
|
||||
progress->end();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
};
|
||||
Vendored
+115
@@ -0,0 +1,115 @@
|
||||
|
||||
ifndef PLATFORM
|
||||
PLATFORM := native
|
||||
endif
|
||||
|
||||
ifneq ($(PLATFORM),$(filter $(PLATFORM),amiga windows-32 windows-64 native native-32 native-64))
|
||||
DUMMY := $(error Unsupported platform $(PLATFORM))
|
||||
endif
|
||||
|
||||
BUILD_DIR := build/$(PLATFORM)
|
||||
MKDIR_DUMMY := $(shell mkdir -p $(BUILD_DIR))
|
||||
|
||||
all: $(BUILD_DIR)/Shrinkler
|
||||
|
||||
# Common flags
|
||||
CFLAGS := -Wall -Wno-sign-compare
|
||||
LFLAGS := -s
|
||||
|
||||
ifdef DEBUG
|
||||
CFLAGS += -g -DDEBUG
|
||||
LFLAGS :=
|
||||
else
|
||||
CFLAGS += -O3
|
||||
endif
|
||||
|
||||
ifdef PROFILE
|
||||
CFLAGS += -fno-inline -fno-inline-functions
|
||||
LFLAGS :=
|
||||
endif
|
||||
|
||||
ifeq ($(PLATFORM),amiga)
|
||||
|
||||
# Amiga build, using GCC and ixemul
|
||||
|
||||
TOOLCHAIN_DIR := toolchain
|
||||
AMIGA_GCC_DIR := $(TOOLCHAIN_DIR)/GCC-4.5.0-m68k-amigaos-cygwin/usr/local/amiga
|
||||
BINUTILS_DIR := $(TOOLCHAIN_DIR)/amiga-binutils
|
||||
INCLUDE_DIR := $(TOOLCHAIN_DIR)/C++include/include
|
||||
LIB_DIR1 := $(TOOLCHAIN_DIR)/C++include/lib
|
||||
LIB_DIR2 := $(AMIGA_GCC_DIR)/lib/gcc/m68k-amigaos/4.5.0
|
||||
LIB_DIR3 := $(TOOLCHAIN_DIR)/ixemul-sdk/lib
|
||||
|
||||
CC := $(AMIGA_GCC_DIR)/bin/m68k-amigaos-g++
|
||||
CFLAGS += -m68000
|
||||
INCLUDE := -I $(INCLUDE_DIR)/c++/4.3.2 -I $(INCLUDE_DIR)/c++/4.3.2/m68k-amigaos -I $(INCLUDE_DIR)
|
||||
|
||||
ASM := $(BINUTILS_DIR)/as
|
||||
ASMFLAGS :=
|
||||
|
||||
LINK := $(BINUTILS_DIR)/ld
|
||||
STARTUP := $(LIB_DIR3)/crt0.o
|
||||
LIBS := -L $(LIB_DIR1) -L $(LIB_DIR2) -L $(LIB_DIR3) -lstdc++ -lgcc -lc
|
||||
|
||||
$(BUILD_DIR)/%.o: %.cpp
|
||||
$(CC) $(CFLAGS) $(INCLUDE) $< -S -o $(@:%.o=%.s)
|
||||
$(ASM) $(ASMFLAGS) $(@:%.o=%.s) -o $@
|
||||
|
||||
else
|
||||
ifeq ($(PLATFORM),windows-32)
|
||||
|
||||
# 32-bit MinGW build
|
||||
|
||||
CC := i686-w64-mingw32-g++
|
||||
LINK := i686-w64-mingw32-g++
|
||||
LFLAGS += -static-libgcc -static-libstdc++
|
||||
|
||||
else
|
||||
ifeq ($(PLATFORM),windows-64)
|
||||
|
||||
# 64-bit MinGW build
|
||||
|
||||
CC := x86_64-w64-mingw32-g++
|
||||
LINK := x86_64-w64-mingw32-g++
|
||||
LFLAGS += -static-libgcc -static-libstdc++
|
||||
|
||||
else
|
||||
|
||||
# Native build
|
||||
|
||||
CC := g++
|
||||
LINK := g++
|
||||
|
||||
ifeq ($(PLATFORM),native-32)
|
||||
CFLAGS += -m32
|
||||
endif
|
||||
|
||||
ifeq ($(PLATFORM),native-64)
|
||||
CFLAGS += -m64
|
||||
endif
|
||||
|
||||
endif
|
||||
endif
|
||||
|
||||
# Common setup for non-Amiga builds
|
||||
|
||||
INCLUDE :=
|
||||
STARTUP :=
|
||||
LIBS :=
|
||||
|
||||
$(BUILD_DIR)/%.o: %.cpp
|
||||
$(CC) $(CFLAGS) $(INCLUDE) $< -c -o $@
|
||||
|
||||
endif
|
||||
|
||||
|
||||
$(BUILD_DIR)/Shrinkler.o: *.h Header1.dat Header1T.dat Header2.dat OverlapHeader.dat OverlapHeaderT.dat MiniHeader.dat
|
||||
|
||||
%.dat: %.bin
|
||||
python -c 'for b in open("$^", "rb").read(): print ("0x%02X," % ord(b)),' > $@
|
||||
|
||||
$(BUILD_DIR)/Shrinkler: $(BUILD_DIR)/Shrinkler.o
|
||||
$(LINK) $(LFLAGS) $(STARTUP) $< $(LIBS) -o $@
|
||||
|
||||
clean:
|
||||
rm -rf build
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
Find repeated strings in a data block.
|
||||
|
||||
Matches are reported from longest to shortest. A match is only reported
|
||||
if it is closer (smaller offset, higher position) than all longer matches.
|
||||
|
||||
Two parameters control the speed/precision tradeoff of the matcher:
|
||||
|
||||
The match_patience parameter controls how many matches outside the current
|
||||
reporting range (between last longer match and current position) are skipped
|
||||
before the matcher gives up finding more matches.
|
||||
|
||||
The max_same_length parameter controls how many matches of the same length
|
||||
are reported. The matches reported will be the closest ones of that length.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <algorithm>
|
||||
#include <queue>
|
||||
#include <functional>
|
||||
|
||||
using std::vector;
|
||||
|
||||
class MatchFinder {
|
||||
// Inputs
|
||||
unsigned char *data;
|
||||
int length;
|
||||
int min_length;
|
||||
int match_patience;
|
||||
int max_same_length;
|
||||
|
||||
// Suffix array
|
||||
vector<int> suffix_array;
|
||||
vector<int> rev_suffix_array;
|
||||
vector<int> longest_common_prefix;
|
||||
|
||||
// Matcher parameters
|
||||
int current_pos;
|
||||
int min_pos;
|
||||
|
||||
// Matcher state
|
||||
int left_index;
|
||||
int left_length;
|
||||
int right_index;
|
||||
int right_length;
|
||||
int current_length;
|
||||
|
||||
// Best matches seen with current length
|
||||
std::priority_queue<int, vector<int>, std::greater<int> > match_buffer;
|
||||
|
||||
struct suffix_compare {
|
||||
MatchFinder* finder;
|
||||
|
||||
suffix_compare(MatchFinder* finder) : finder(finder) {}
|
||||
|
||||
// Compare suffixes starting at positions a and b
|
||||
bool operator()(int a, int b) {
|
||||
vector<int>& same = finder->rev_suffix_array;
|
||||
unsigned char *data = finder->data;
|
||||
if (a == b) return false;
|
||||
if (data[a] == data[b]) {
|
||||
// Skip stretch of equal bytes
|
||||
int skip = std::min(same[a], same[b]);
|
||||
a += skip;
|
||||
b += skip;
|
||||
}
|
||||
int until_end = finder->length - std::max(a,b);
|
||||
for (int i = 0 ; i < until_end ; i++) {
|
||||
if (data[a + i] != data[b + i]) {
|
||||
return data[a + i] < data[b + i];
|
||||
}
|
||||
}
|
||||
return a < b;
|
||||
}
|
||||
};
|
||||
|
||||
// Quick'n'dirty suffix array construction:
|
||||
// plain sorting with accelleration of same-value blocks
|
||||
void make_suffix_array() {
|
||||
// Temporary same-value block accelleration array
|
||||
rev_suffix_array.resize(length + 1);
|
||||
int count = 0;
|
||||
char c = 0;
|
||||
rev_suffix_array[length] = 0;
|
||||
for (int i = length - 1 ; i >= 0 ; i--) {
|
||||
if (data[i] != c) count = 0;
|
||||
rev_suffix_array[i] = ++count;
|
||||
c = data[i];
|
||||
}
|
||||
|
||||
// Compute suffix array
|
||||
suffix_array.resize(length + 1);
|
||||
for (int i = 0 ; i <= length ; i++) {
|
||||
suffix_array[i] = i;
|
||||
}
|
||||
std::sort(&suffix_array[0], &suffix_array[length], suffix_compare(this));
|
||||
|
||||
// Compute reverse suffix array
|
||||
for (int i = 0 ; i <= length ; i++) {
|
||||
rev_suffix_array[suffix_array[i]] = i;
|
||||
}
|
||||
|
||||
// Compute LCP array
|
||||
longest_common_prefix.resize(length + 1);
|
||||
longest_common_prefix[0] = 0;
|
||||
longest_common_prefix[length] = 0;
|
||||
int h = 0;
|
||||
for (int i = 0 ; i < length ; i++) {
|
||||
int r = rev_suffix_array[i];
|
||||
if (r > 0) {
|
||||
int j = suffix_array[r - 1];
|
||||
while (data[i + h] == data[j + h]) {
|
||||
h = h + 1;
|
||||
}
|
||||
longest_common_prefix[r] = h;
|
||||
if (h > 0) h = h - 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void extend_left() {
|
||||
int iter = 0;
|
||||
while (left_length >= min_length) {
|
||||
left_length = std::min(left_length, longest_common_prefix[left_index]);
|
||||
int pos = suffix_array[--left_index];
|
||||
if (pos < current_pos && pos >= min_pos) break;
|
||||
if (++iter > match_patience) left_length = 0;
|
||||
}
|
||||
}
|
||||
|
||||
void extend_right() {
|
||||
int iter = 0;
|
||||
while (right_length >= min_length) {
|
||||
right_length = std::min(right_length, longest_common_prefix[++right_index]);
|
||||
int pos = suffix_array[right_index];
|
||||
if (pos < current_pos && pos >= min_pos) break;
|
||||
if (++iter > match_patience) right_length = 0;
|
||||
}
|
||||
}
|
||||
|
||||
int next_length() {
|
||||
return std::max(left_length, right_length);
|
||||
}
|
||||
|
||||
public:
|
||||
MatchFinder(unsigned char *data, int length, int min_length, int match_patience, int max_same_length) :
|
||||
data(data), length(length), min_length(min_length), match_patience(match_patience), max_same_length(max_same_length) {
|
||||
make_suffix_array();
|
||||
reset();
|
||||
}
|
||||
|
||||
void reset() {
|
||||
}
|
||||
|
||||
// Start finding matches between strings starting at pos and earlier strings.
|
||||
void beginMatching(int pos) {
|
||||
current_pos = pos;
|
||||
min_pos = 0;
|
||||
|
||||
left_index = rev_suffix_array[pos];
|
||||
left_length = length;
|
||||
extend_left();
|
||||
right_index = rev_suffix_array[pos];
|
||||
right_length = length;
|
||||
extend_right();
|
||||
}
|
||||
|
||||
// Report next match. Returns whether a match was found.
|
||||
bool nextMatch(int *match_pos_out, int *match_length_out) {
|
||||
if (match_buffer.empty()) {
|
||||
// Fill match buffer
|
||||
current_length = next_length();
|
||||
if (current_length < min_length) return false;
|
||||
int new_min_pos = min_pos;
|
||||
do {
|
||||
int match_pos;
|
||||
if (left_length > right_length) {
|
||||
match_pos = suffix_array[left_index];
|
||||
extend_left();
|
||||
} else {
|
||||
match_pos = suffix_array[right_index];
|
||||
extend_right();
|
||||
}
|
||||
new_min_pos = std::max(new_min_pos, match_pos);
|
||||
if (match_buffer.size() < max_same_length) {
|
||||
match_buffer.push(match_pos);
|
||||
} else {
|
||||
if (match_pos > match_buffer.top()) {
|
||||
match_buffer.pop();
|
||||
match_buffer.push(match_pos);
|
||||
}
|
||||
min_pos = match_buffer.top();
|
||||
}
|
||||
} while (next_length() == current_length);
|
||||
assert(!match_buffer.empty());
|
||||
min_pos = new_min_pos;
|
||||
}
|
||||
|
||||
*match_length_out = current_length;
|
||||
*match_pos_out = match_buffer.top();
|
||||
match_buffer.pop();
|
||||
assert(*match_pos_out < current_pos);
|
||||
return true;
|
||||
}
|
||||
};
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
; Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
; auto wb\MiniHeader\MiniHeader_End\
|
||||
|
||||
INIT_ONE_PROB = $8000
|
||||
ADJUST_SHIFT = 4
|
||||
SINGLE_BIT_CONTEXTS = 1
|
||||
DUMMY_CONTEXT_OFFSET = 0
|
||||
|
||||
LIB_VERSION = 20
|
||||
CacheClearU = -636
|
||||
|
||||
align 0,4
|
||||
|
||||
MiniHeader:
|
||||
move.l (a3),d2
|
||||
lsl.l #2,d2
|
||||
move.l d2,a1
|
||||
addq.l #4,a1
|
||||
pea.l (a1)
|
||||
ContextOffsetInstr:
|
||||
lea.l MiniHeader_End+DUMMY_CONTEXT_OFFSET+(32768/8)*2(pc),a2
|
||||
moveq.l #0,d1
|
||||
.init: move.w #INIT_ONE_PROB,-(a2)
|
||||
addq.w #8,d1
|
||||
bpl.b .init
|
||||
|
||||
move.l a2,a4
|
||||
Depack:
|
||||
; A4 = Packed data End
|
||||
; A1 = Target
|
||||
; A2 = Contexts
|
||||
|
||||
; D1 = $00008000
|
||||
; Lowest bit of D2 = 0
|
||||
swap.w d1
|
||||
moveq.l #1,d3
|
||||
moveq.l #0,d6
|
||||
.lit:
|
||||
addq.b #1,d6
|
||||
.getlit:
|
||||
bsr.b GetBit
|
||||
addx.b d6,d6
|
||||
bcc.b .getlit
|
||||
move.b d6,(a1)+
|
||||
.switch:
|
||||
bsr.b GetKind
|
||||
bcc.b .lit
|
||||
.ref:
|
||||
moveq.l #-1,d6
|
||||
bsr.b GetBit
|
||||
bcs.b .sameoffset
|
||||
.newref:
|
||||
moveq.l #3,d6
|
||||
bsr.b GetNumber
|
||||
moveq.l #2,d5
|
||||
sub.l d7,d5
|
||||
beq.b .end
|
||||
.sameoffset:
|
||||
moveq.l #4,d6
|
||||
bsr.b GetNumber
|
||||
.copyloop:
|
||||
move.b (a1,d5.l),(a1)+
|
||||
subq.l #1,d7
|
||||
bne.b .copyloop
|
||||
.afterref:
|
||||
bsr.b GetKind
|
||||
bcc.b .lit
|
||||
bra.b .newref
|
||||
|
||||
.end:
|
||||
move.l $4.w,a6
|
||||
cmp.w #37,LIB_VERSION(a6)
|
||||
blt.b not204
|
||||
jmp CacheClearU(a6)
|
||||
|
||||
|
||||
GetKind:
|
||||
move.l a1,d4
|
||||
moveq.l #1,d6
|
||||
and.l d4,d6
|
||||
lsl.w #8,d6
|
||||
bra.b GetBit
|
||||
|
||||
GetNumber:
|
||||
; D6 = Number context
|
||||
|
||||
; Out: Number in D7
|
||||
lsl.w #8,d6
|
||||
.numberloop:
|
||||
addq.b #2,d6
|
||||
bsr.b GetBit
|
||||
bcs.b .numberloop
|
||||
moveq.l #1,d7
|
||||
subq.b #1,d6
|
||||
.bitsloop:
|
||||
bsr.b GetBit
|
||||
addx.l d7,d7
|
||||
subq.b #2,d6
|
||||
bcc.b .bitsloop
|
||||
not204: rts
|
||||
|
||||
; D6 = Bit context
|
||||
|
||||
; D1 = Input bit buffer
|
||||
; D2 = Range value
|
||||
; D3 = Interval size
|
||||
|
||||
; Out: Bit in C and X
|
||||
|
||||
readbit:
|
||||
add.l d1,d1
|
||||
bne.b nonewword
|
||||
move.l -(a4),d1
|
||||
addx.l d1,d1
|
||||
nonewword:
|
||||
addx.w d2,d2
|
||||
add.w d3,d3
|
||||
GetBit:
|
||||
tst.w d3
|
||||
bpl.b readbit
|
||||
|
||||
lea.l SINGLE_BIT_CONTEXTS*2(a2,d6.l),a5
|
||||
add.l d6,a5
|
||||
move.w (a5),d4
|
||||
; D4 = One prob
|
||||
|
||||
lsr.w #ADJUST_SHIFT,d4
|
||||
sub.w d4,(a5)
|
||||
add.w (a5),d4
|
||||
|
||||
mulu.w d3,d4
|
||||
swap.w d4
|
||||
|
||||
sub.w d4,d2
|
||||
blo.b .one
|
||||
.zero:
|
||||
; oneprob = oneprob * (1 - adjust) = oneprob - oneprob * adjust
|
||||
sub.w d4,d3
|
||||
; 0 in C and X
|
||||
rts
|
||||
.one:
|
||||
; onebrob = 1 - (1 - oneprob) * (1 - adjust) = oneprob - oneprob * adjust + adjust
|
||||
add.w #$ffff>>ADJUST_SHIFT,(a5)
|
||||
move.w d4,d3
|
||||
add.w d4,d2
|
||||
; 1 in C and X
|
||||
rts
|
||||
|
||||
align 0,4
|
||||
|
||||
MiniHeader_End:
|
||||
|
||||
printv MiniHeader_End-MiniHeader
|
||||
printt
|
||||
printv ContextOffsetInstr+2-MiniHeader
|
||||
BIN
Binary file not shown.
+1
@@ -0,0 +1 @@
|
||||
0x24, 0x13, 0xE5, 0x8A, 0x22, 0x42, 0x58, 0x89, 0x48, 0x51, 0x45, 0xFA, 0x20, 0xA8, 0x72, 0x00, 0x35, 0x3C, 0x80, 0x00, 0x50, 0x41, 0x6A, 0xF8, 0x28, 0x4A, 0x48, 0x41, 0x76, 0x01, 0x7C, 0x00, 0x52, 0x06, 0x61, 0x68, 0xDD, 0x06, 0x64, 0xFA, 0x12, 0xC6, 0x61, 0x34, 0x64, 0xF2, 0x7C, 0xFF, 0x61, 0x5A, 0x65, 0x0A, 0x7C, 0x03, 0x61, 0x32, 0x7A, 0x02, 0x9A, 0x87, 0x67, 0x12, 0x7C, 0x04, 0x61, 0x28, 0x12, 0xF1, 0x58, 0x00, 0x53, 0x87, 0x66, 0xF8, 0x61, 0x14, 0x64, 0xD2, 0x60, 0xE4, 0x2C, 0x78, 0x00, 0x04, 0x0C, 0x6E, 0x00, 0x25, 0x00, 0x14, 0x6D, 0x22, 0x4E, 0xEE, 0xFD, 0x84, 0x28, 0x09, 0x7C, 0x01, 0xCC, 0x84, 0xE1, 0x4E, 0x60, 0x22, 0xE1, 0x4E, 0x54, 0x06, 0x61, 0x1C, 0x65, 0xFA, 0x7E, 0x01, 0x53, 0x06, 0x61, 0x14, 0xDF, 0x87, 0x55, 0x06, 0x64, 0xF8, 0x4E, 0x75, 0xD2, 0x81, 0x66, 0x04, 0x22, 0x24, 0xD3, 0x81, 0xD5, 0x42, 0xD6, 0x43, 0x4A, 0x43, 0x6A, 0xF0, 0x4B, 0xF2, 0x68, 0x02, 0xDB, 0xC6, 0x38, 0x15, 0xE8, 0x4C, 0x99, 0x55, 0xD8, 0x55, 0xC8, 0xC3, 0x48, 0x44, 0x94, 0x44, 0x65, 0x04, 0x96, 0x44, 0x4E, 0x75, 0x06, 0x55, 0x0F, 0xFF, 0x36, 0x04, 0xD4, 0x44, 0x4E, 0x75,
|
||||
+230
@@ -0,0 +1,230 @@
|
||||
; Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
TEXT = 0
|
||||
|
||||
; auto wb\OverlapHeader\OverlapHeader_End\
|
||||
|
||||
INIT_ONE_PROB = $8000
|
||||
ADJUST_SHIFT = 4
|
||||
SINGLE_BIT_CONTEXTS = 1
|
||||
NUM_CONTEXTS = 1536
|
||||
DUMMY_TEXT_LENGTH = 0
|
||||
|
||||
; Exec
|
||||
LIB_VERSION = 20
|
||||
OldOpenLibrary = -408
|
||||
CloseLibrary = -414
|
||||
CacheClearU = -636
|
||||
|
||||
; Dos
|
||||
Write = -48
|
||||
Output = -60
|
||||
|
||||
|
||||
align 0,4
|
||||
OverlapHeader:
|
||||
move.l (a3),d2
|
||||
lsl.l #2,d2
|
||||
move.l d2,a3
|
||||
move.l $4.w,a6
|
||||
|
||||
if TEXT
|
||||
movem.l d0/a0,-(a7)
|
||||
TextLengthInstr:
|
||||
move.l #DUMMY_TEXT_LENGTH,d3
|
||||
lea.l DosName(pc),a1
|
||||
jsr OldOpenLibrary(a6)
|
||||
move.l d0,a6
|
||||
jsr Output(a6)
|
||||
move.l d0,d1
|
||||
beq.b noout
|
||||
TextOffsetInstr:
|
||||
lea.l OverlapHeader_End(pc),a0
|
||||
move.l a0,d2
|
||||
jsr Write(a6)
|
||||
noout: move.l a6,a1
|
||||
move.l $4.w,a6
|
||||
jsr CloseLibrary(a6)
|
||||
movem.l (a7)+,d0/a0
|
||||
endc
|
||||
|
||||
; Init range decoder state
|
||||
moveq.l #1,d1
|
||||
ror.l #1,d1
|
||||
moveq.l #1,d3
|
||||
; Lowest bit of D2 = 0
|
||||
|
||||
move.l a3,a2
|
||||
HunkLoop:
|
||||
; Move packed data to end of hunk
|
||||
move.l a2,a4
|
||||
add.l -(a4),a4
|
||||
lea.l 4(a2),a1
|
||||
move.l (a1),d6
|
||||
.move: move.l (a1,d6.l),-(a4)
|
||||
subq.l #4,d6
|
||||
bgt.b .move
|
||||
; A4 = Start of packed data hunk
|
||||
; A1 = Hunk Data Destination
|
||||
|
||||
moveq.l #NUM_CONTEXTS>>4,d6
|
||||
lsl.l #4,d6
|
||||
.init: move.w #INIT_ONE_PROB,-(a7)
|
||||
subq.w #1,d6
|
||||
bne.b .init
|
||||
|
||||
; moveq.l #0,d6
|
||||
.lit:
|
||||
addq.b #1,d6
|
||||
.getlit:
|
||||
bsr.b GetBit
|
||||
addx.b d6,d6
|
||||
bcc.b .getlit
|
||||
move.b d6,(a1)+
|
||||
.switch:
|
||||
bsr.b GetKind
|
||||
bcc.b .lit
|
||||
.ref:
|
||||
moveq.l #-1,d6
|
||||
bsr.b GetBit
|
||||
bcs.b .sameoffset
|
||||
.newref:
|
||||
moveq.l #3,d6
|
||||
bsr.b GetNumber
|
||||
moveq.l #2,d5
|
||||
sub.l d7,d5
|
||||
beq.b .hunkend
|
||||
.sameoffset:
|
||||
moveq.l #4,d6
|
||||
bsr.b GetNumber
|
||||
.copyloop:
|
||||
move.b (a1,d5.l),(a1)+
|
||||
subq.l #1,d7
|
||||
bne.b .copyloop
|
||||
.afterref:
|
||||
bsr.b GetKind
|
||||
bcc.b .lit
|
||||
bra.b .newref
|
||||
.hunkend:
|
||||
|
||||
; Relocs
|
||||
move.l a3,d5
|
||||
RelocHunk:
|
||||
addq.l #4,d5
|
||||
move.l a2,a1
|
||||
.relocloop:
|
||||
moveq.l #5,d6
|
||||
bsr.b GetNumber
|
||||
add.l d7,a1
|
||||
lsr.l #2,d7
|
||||
beq.b NextRelocHunk
|
||||
add.l d5,(a1)
|
||||
bra.b .relocloop
|
||||
|
||||
NextRelocHunk:
|
||||
move.l d5,a1
|
||||
move.l -(a1),d5
|
||||
lsl.l #2,d5
|
||||
bne.b RelocHunk
|
||||
NextHunk:
|
||||
lea.l NUM_CONTEXTS*2(a7),a7
|
||||
|
||||
move.l (a2),d4
|
||||
lsl.l #2,d4
|
||||
move.l d4,a2
|
||||
bne.b HunkLoop
|
||||
End:
|
||||
cmp.w #37,LIB_VERSION(a6)
|
||||
blt.b .not204
|
||||
jsr CacheClearU(a6)
|
||||
.not204:
|
||||
jmp 4(a3)
|
||||
|
||||
if TEXT
|
||||
DosName:
|
||||
dc.b "dos.library",0
|
||||
endc
|
||||
|
||||
|
||||
GetKind:
|
||||
move.l a1,d4
|
||||
moveq.l #1,d6
|
||||
and.l d4,d6
|
||||
lsl.w #8,d6
|
||||
GetBit: bra.b GetBitInner
|
||||
|
||||
GetNumber:
|
||||
; D6 = Number context
|
||||
|
||||
; Out: Number in D7
|
||||
lsl.w #8,d6
|
||||
.numberloop:
|
||||
addq.b #2,d6
|
||||
bsr.b GetBitInner
|
||||
bcs.b .numberloop
|
||||
moveq.l #1,d7
|
||||
subq.b #1,d6
|
||||
.bitsloop:
|
||||
bsr.b GetBitInner
|
||||
addx.l d7,d7
|
||||
subq.b #2,d6
|
||||
bcc.b .bitsloop
|
||||
rts
|
||||
|
||||
; D6 = Bit context
|
||||
|
||||
; D1 = Input bit buffer
|
||||
; D2 = Range value
|
||||
; D3 = Interval size
|
||||
|
||||
; Out: Bit in C and X
|
||||
|
||||
readbit:
|
||||
add.l d1,d1
|
||||
bne.b nonewword
|
||||
move.l (a4)+,d1
|
||||
addx.l d1,d1
|
||||
nonewword:
|
||||
addx.w d2,d2
|
||||
add.w d3,d3
|
||||
GetBitInner:
|
||||
tst.w d3
|
||||
bpl.b readbit
|
||||
|
||||
lea.l 4+SINGLE_BIT_CONTEXTS*2(a7,d6.l),a5
|
||||
add.l d6,a5
|
||||
move.w (a5),d4
|
||||
; D4 = One prob
|
||||
|
||||
lsr.w #ADJUST_SHIFT,d4
|
||||
sub.w d4,(a5)
|
||||
add.w (a5),d4
|
||||
|
||||
mulu.w d3,d4
|
||||
swap.w d4
|
||||
|
||||
sub.w d4,d2
|
||||
blo.b .one
|
||||
.zero:
|
||||
; oneprob = oneprob * (1 - adjust) = oneprob - oneprob * adjust
|
||||
sub.w d4,d3
|
||||
; 0 in C and X
|
||||
rts
|
||||
.one:
|
||||
; onebrob = 1 - (1 - oneprob) * (1 - adjust) = oneprob - oneprob * adjust + adjust
|
||||
add.w #$ffff>>ADJUST_SHIFT,(a5)
|
||||
move.w d4,d3
|
||||
add.w d4,d2
|
||||
; 1 in C and X
|
||||
rts
|
||||
|
||||
align 0,4
|
||||
|
||||
OverlapHeader_End:
|
||||
|
||||
printv OverlapHeader_End-OverlapHeader
|
||||
if TEXT
|
||||
printt
|
||||
printv TextLengthInstr+2-OverlapHeader
|
||||
printv TextOffsetInstr+2-OverlapHeader
|
||||
endc
|
||||
BIN
Binary file not shown.
+1
@@ -0,0 +1 @@
|
||||
0x24, 0x13, 0xE5, 0x8A, 0x26, 0x42, 0x2C, 0x78, 0x00, 0x04, 0x72, 0x01, 0xE2, 0x99, 0x76, 0x01, 0x24, 0x4B, 0x28, 0x4A, 0xD9, 0xE4, 0x43, 0xEA, 0x00, 0x04, 0x2C, 0x11, 0x29, 0x31, 0x68, 0x00, 0x59, 0x86, 0x6E, 0xF8, 0x7C, 0x60, 0xE9, 0x8E, 0x3F, 0x3C, 0x80, 0x00, 0x53, 0x46, 0x66, 0xF8, 0x52, 0x06, 0x61, 0x6C, 0xDD, 0x06, 0x64, 0xFA, 0x12, 0xC6, 0x61, 0x5C, 0x64, 0xF2, 0x7C, 0xFF, 0x61, 0x5E, 0x65, 0x0A, 0x7C, 0x03, 0x61, 0x5A, 0x7A, 0x02, 0x9A, 0x87, 0x67, 0x12, 0x7C, 0x04, 0x61, 0x50, 0x12, 0xF1, 0x58, 0x00, 0x53, 0x87, 0x66, 0xF8, 0x61, 0x3C, 0x64, 0xD2, 0x60, 0xE4, 0x2A, 0x0B, 0x58, 0x85, 0x22, 0x4A, 0x7C, 0x05, 0x61, 0x38, 0xD3, 0xC7, 0xE4, 0x8F, 0x67, 0x04, 0xDB, 0x91, 0x60, 0xF2, 0x22, 0x45, 0x2A, 0x21, 0xE5, 0x8D, 0x66, 0xE6, 0x4F, 0xEF, 0x0C, 0x00, 0x28, 0x12, 0xE5, 0x8C, 0x24, 0x44, 0x66, 0x8A, 0x0C, 0x6E, 0x00, 0x25, 0x00, 0x14, 0x6D, 0x04, 0x4E, 0xAE, 0xFD, 0x84, 0x4E, 0xEB, 0x00, 0x04, 0x28, 0x09, 0x7C, 0x01, 0xCC, 0x84, 0xE1, 0x4E, 0x60, 0x22, 0xE1, 0x4E, 0x54, 0x06, 0x61, 0x1C, 0x65, 0xFA, 0x7E, 0x01, 0x53, 0x06, 0x61, 0x14, 0xDF, 0x87, 0x55, 0x06, 0x64, 0xF8, 0x4E, 0x75, 0xD2, 0x81, 0x66, 0x04, 0x22, 0x1C, 0xD3, 0x81, 0xD5, 0x42, 0xD6, 0x43, 0x4A, 0x43, 0x6A, 0xF0, 0x4B, 0xF7, 0x68, 0x06, 0xDB, 0xC6, 0x38, 0x15, 0xE8, 0x4C, 0x99, 0x55, 0xD8, 0x55, 0xC8, 0xC3, 0x48, 0x44, 0x94, 0x44, 0x65, 0x04, 0x96, 0x44, 0x4E, 0x75, 0x06, 0x55, 0x0F, 0xFF, 0x36, 0x04, 0xD4, 0x44, 0x4E, 0x75,
|
||||
BIN
Binary file not shown.
+1
@@ -0,0 +1 @@
|
||||
0x24, 0x13, 0xE5, 0x8A, 0x26, 0x42, 0x2C, 0x78, 0x00, 0x04, 0x48, 0xE7, 0x80, 0x80, 0x26, 0x3C, 0x00, 0x00, 0x00, 0x00, 0x43, 0xFA, 0x00, 0xB6, 0x4E, 0xAE, 0xFE, 0x68, 0x2C, 0x40, 0x4E, 0xAE, 0xFF, 0xC4, 0x22, 0x00, 0x67, 0x0A, 0x41, 0xFA, 0x01, 0x04, 0x24, 0x08, 0x4E, 0xAE, 0xFF, 0xD0, 0x22, 0x4E, 0x2C, 0x78, 0x00, 0x04, 0x4E, 0xAE, 0xFE, 0x62, 0x4C, 0xDF, 0x01, 0x01, 0x72, 0x01, 0xE2, 0x99, 0x76, 0x01, 0x24, 0x4B, 0x28, 0x4A, 0xD9, 0xE4, 0x43, 0xEA, 0x00, 0x04, 0x2C, 0x11, 0x29, 0x31, 0x68, 0x00, 0x59, 0x86, 0x6E, 0xF8, 0x7C, 0x60, 0xE9, 0x8E, 0x3F, 0x3C, 0x80, 0x00, 0x53, 0x46, 0x66, 0xF8, 0x52, 0x06, 0x61, 0x78, 0xDD, 0x06, 0x64, 0xFA, 0x12, 0xC6, 0x61, 0x68, 0x64, 0xF2, 0x7C, 0xFF, 0x61, 0x6A, 0x65, 0x0A, 0x7C, 0x03, 0x61, 0x66, 0x7A, 0x02, 0x9A, 0x87, 0x67, 0x12, 0x7C, 0x04, 0x61, 0x5C, 0x12, 0xF1, 0x58, 0x00, 0x53, 0x87, 0x66, 0xF8, 0x61, 0x48, 0x64, 0xD2, 0x60, 0xE4, 0x2A, 0x0B, 0x58, 0x85, 0x22, 0x4A, 0x7C, 0x05, 0x61, 0x44, 0xD3, 0xC7, 0xE4, 0x8F, 0x67, 0x04, 0xDB, 0x91, 0x60, 0xF2, 0x22, 0x45, 0x2A, 0x21, 0xE5, 0x8D, 0x66, 0xE6, 0x4F, 0xEF, 0x0C, 0x00, 0x28, 0x12, 0xE5, 0x8C, 0x24, 0x44, 0x66, 0x8A, 0x0C, 0x6E, 0x00, 0x25, 0x00, 0x14, 0x6D, 0x04, 0x4E, 0xAE, 0xFD, 0x84, 0x4E, 0xEB, 0x00, 0x04, 0x64, 0x6F, 0x73, 0x2E, 0x6C, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x00, 0x28, 0x09, 0x7C, 0x01, 0xCC, 0x84, 0xE1, 0x4E, 0x60, 0x22, 0xE1, 0x4E, 0x54, 0x06, 0x61, 0x1C, 0x65, 0xFA, 0x7E, 0x01, 0x53, 0x06, 0x61, 0x14, 0xDF, 0x87, 0x55, 0x06, 0x64, 0xF8, 0x4E, 0x75, 0xD2, 0x81, 0x66, 0x04, 0x22, 0x1C, 0xD3, 0x81, 0xD5, 0x42, 0xD6, 0x43, 0x4A, 0x43, 0x6A, 0xF0, 0x4B, 0xF7, 0x68, 0x06, 0xDB, 0xC6, 0x38, 0x15, 0xE8, 0x4C, 0x99, 0x55, 0xD8, 0x55, 0xC8, 0xC3, 0x48, 0x44, 0x94, 0x44, 0x65, 0x04, 0x96, 0x44, 0x4E, 0x75, 0x06, 0x55, 0x0F, 0xFF, 0x36, 0x04, 0xD4, 0x44, 0x4E, 0x75,
|
||||
Vendored
+133
@@ -0,0 +1,133 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
Pack a data block in multiple iterations, reporting progress along the way.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "RangeCoder.h"
|
||||
#include "MatchFinder.h"
|
||||
#include "CountingCoder.h"
|
||||
#include "SizeMeasuringCoder.h"
|
||||
#include "LZEncoder.h"
|
||||
#include "LZParser.h"
|
||||
|
||||
struct PackParams {
|
||||
int iterations;
|
||||
int length_margin;
|
||||
int skip_length;
|
||||
int match_patience;
|
||||
int max_same_length;
|
||||
};
|
||||
|
||||
class PackProgress : public LZProgress {
|
||||
int size;
|
||||
int steps;
|
||||
int next_step_threshold;
|
||||
int textlength;
|
||||
|
||||
void print() {
|
||||
textlength = printf("[%d.%d%%]", steps / 10, steps % 10);
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
void rewind() {
|
||||
printf("\033[%dD", textlength);
|
||||
}
|
||||
public:
|
||||
virtual void begin(int size) {
|
||||
this->size = size;
|
||||
steps = 0;
|
||||
next_step_threshold = size / 1000;
|
||||
print();
|
||||
}
|
||||
|
||||
virtual void update(int pos) {
|
||||
if (pos < next_step_threshold) return;
|
||||
while (pos >= next_step_threshold) {
|
||||
steps += 1;
|
||||
next_step_threshold = (long long) size * (steps + 1) / 1000;
|
||||
}
|
||||
rewind();
|
||||
print();
|
||||
}
|
||||
|
||||
virtual void end() {
|
||||
rewind();
|
||||
printf("\033[K");
|
||||
fflush(stdout);
|
||||
}
|
||||
};
|
||||
|
||||
class NoProgress : public LZProgress {
|
||||
public:
|
||||
virtual void begin(int size) {
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
virtual void update(int pos) {
|
||||
}
|
||||
|
||||
virtual void end() {
|
||||
}
|
||||
};
|
||||
|
||||
void packData(unsigned char *data, int data_length, int zero_padding, PackParams *params, Coder *result_coder, RefEdgeFactory *edge_factory, bool show_progress) {
|
||||
MatchFinder finder(data, data_length, 2, params->match_patience, params->max_same_length);
|
||||
LZParser parser(data, data_length, zero_padding, finder, params->length_margin, params->skip_length, edge_factory);
|
||||
int real_size = 0;
|
||||
int best_size = 999999999;
|
||||
int best_result = 0;
|
||||
vector<LZParseResult> results(2);
|
||||
CountingCoder *counting_coder = new CountingCoder(LZEncoder::NUM_CONTEXTS);
|
||||
LZProgress *progress;
|
||||
if (show_progress) {
|
||||
progress = new PackProgress();
|
||||
} else {
|
||||
progress = new NoProgress();
|
||||
}
|
||||
printf("%8d", data_length);
|
||||
for (int i = 0 ; i < params->iterations ; i++) {
|
||||
printf(" ");
|
||||
|
||||
// Parse data into LZ symbols
|
||||
LZParseResult& result = results[1 - best_result];
|
||||
Coder *measurer = new SizeMeasuringCoder(counting_coder);
|
||||
measurer->setNumberContexts(LZEncoder::NUMBER_CONTEXT_OFFSET, LZEncoder::NUM_NUMBER_CONTEXTS, data_length);
|
||||
finder.reset();
|
||||
result = parser.parse(LZEncoder(measurer), progress);
|
||||
|
||||
// Encode result using adaptive range coding
|
||||
vector<unsigned> dummy_result;
|
||||
RangeCoder *range_coder = new RangeCoder(LZEncoder::NUM_CONTEXTS, dummy_result);
|
||||
real_size = result.encode(LZEncoder(range_coder));
|
||||
range_coder->finish();
|
||||
delete range_coder;
|
||||
|
||||
// Choose if best
|
||||
if (real_size < best_size) {
|
||||
best_result = 1 - best_result;
|
||||
best_size = real_size;
|
||||
}
|
||||
|
||||
// Print size
|
||||
printf("%14.3f", real_size / (double) (8 << Coder::BIT_PRECISION));
|
||||
|
||||
// Count symbol frequencies
|
||||
CountingCoder *new_counting_coder = new CountingCoder(LZEncoder::NUM_CONTEXTS);
|
||||
result.encode(LZEncoder(counting_coder));
|
||||
|
||||
// New size measurer based on frequencies
|
||||
CountingCoder *old_counting_coder = counting_coder;
|
||||
counting_coder = new CountingCoder(old_counting_coder, new_counting_coder);
|
||||
delete old_counting_coder;
|
||||
delete new_counting_coder;
|
||||
}
|
||||
delete progress;
|
||||
delete counting_coder;
|
||||
|
||||
results[best_result].encode(LZEncoder(result_coder));
|
||||
}
|
||||
Vendored
+62
@@ -0,0 +1,62 @@
|
||||
|
||||
Shrinkler executable file compressor for Amiga by Blueberry
|
||||
|
||||
Designed for maximum compression of Amiga 64k and 4k intros, and
|
||||
everything in between.
|
||||
|
||||
Executables for different platforms are available in their respective
|
||||
subdirectories. The output executables are compatible with all Amiga CPUs
|
||||
and kickstarts.
|
||||
|
||||
Run with no arguments for a list of options. For the options controlling
|
||||
compression efficiency, higher values generally result in better
|
||||
compression, at the cost of higher time and/or memory requirements.
|
||||
|
||||
|
||||
History:
|
||||
|
||||
2015-01-18: Version 4.4. Optimizations galore:
|
||||
New match finder based on a suffix array.
|
||||
New reference edge map based on a cuckoo hash table.
|
||||
Pre-compute number encoding sizes for faster estimation.
|
||||
Recycle references edges to save alloc/dealloc overhead.
|
||||
Updated defaults to take advantage of speed increase.
|
||||
Data file compression mode with decompression source.
|
||||
Fixed broken progress output for big files.
|
||||
Do not crash if text file could not be opened.
|
||||
|
||||
2015-01-05: Version 4.3. Minor fixes:
|
||||
Usage information adjusted to fit within 77 columns.
|
||||
References discarded metric computed properly.
|
||||
First progress step is at 0.1% rather than 1.0%.
|
||||
Option to omit progress output (for non-ANSI consoles).
|
||||
Source changes for easier compilation with MSVC.
|
||||
|
||||
2014-12-16: Version 4.2. For memory-efficient decrunching:
|
||||
Option to overlap compressed and decompressed data.
|
||||
Print memory overhead during and after decrunching.
|
||||
Verifier accepts partially filled hunks.
|
||||
|
||||
2014-02-08: Version 4.1. Bug fixes and new features:
|
||||
Fixed some bugs in the range coder.
|
||||
Fixed handling of very small first hunk.
|
||||
Added internal verifier to check correctness of output.
|
||||
Print helpful text when encountering an internal error.
|
||||
Better error message when running out of memory.
|
||||
Set output file to be executable.
|
||||
New options to print text from an argument or file.
|
||||
New option to flash a hardware register during decrunching.
|
||||
|
||||
2014-01-05: Version 4.0. First public release with new name.
|
||||
|
||||
1999 - 2012: Various public and internal versions.
|
||||
|
||||
|
||||
Source code available from https://bitbucket.org/askeksa/shrinkler
|
||||
|
||||
|
||||
For questions and comments, visit the ADA forum at
|
||||
|
||||
http://ada.untergrund.net/?p=boardthread&id=264
|
||||
|
||||
or write to blueberry at loonies dot dk.
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
An entropy coder based on range coding.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <algorithm>
|
||||
#include <vector>
|
||||
|
||||
using std::fill;
|
||||
using std::vector;
|
||||
|
||||
#include "Coder.h"
|
||||
|
||||
#ifndef ADJUST_SHIFT
|
||||
#define ADJUST_SHIFT 4
|
||||
#endif
|
||||
|
||||
class RangeCoder : public Coder {
|
||||
vector<unsigned short> contexts;
|
||||
vector<unsigned>& out;
|
||||
int dest_bit;
|
||||
unsigned intervalsize;
|
||||
unsigned intervalmin;
|
||||
|
||||
static int sizetable[128];
|
||||
static bool sizetable_init;
|
||||
|
||||
static bool init_sizetable() {
|
||||
for (int i = 0 ; i < 128 ; i++) {
|
||||
sizetable[i] = (int) floor(0.5 + (8.0 - log((double) (128 + i)) / log(2.0)) * (1 << BIT_PRECISION));
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void addBit() {
|
||||
int pos = dest_bit;
|
||||
int longpos;
|
||||
int bitmask;
|
||||
do {
|
||||
pos--;
|
||||
if (pos < 0) return;
|
||||
longpos = pos >> 5;
|
||||
bitmask = 0x80000000 >> (pos & 31);
|
||||
while (longpos >= out.size()) {
|
||||
out.push_back(0);
|
||||
}
|
||||
out[longpos] ^= bitmask;
|
||||
} while ((out[longpos] & bitmask) == 0);
|
||||
}
|
||||
|
||||
public:
|
||||
RangeCoder(int n_contexts, vector<unsigned>& out) : out(out) {
|
||||
contexts.resize(n_contexts, 0x8000);
|
||||
dest_bit = -1;
|
||||
intervalsize = 0x8000;
|
||||
intervalmin = 0;
|
||||
out.clear();
|
||||
}
|
||||
|
||||
virtual int code(int context_index, int bit) {
|
||||
assert(context_index < contexts.size());
|
||||
assert(bit == 0 || bit == 1);
|
||||
int size_before = (dest_bit << BIT_PRECISION) + sizetable[(intervalsize - 0x8000) >> 8];
|
||||
unsigned prob = contexts[context_index];
|
||||
unsigned threshold = (intervalsize * prob) >> 16;
|
||||
unsigned new_prob;
|
||||
if (!bit) {
|
||||
// Zero
|
||||
intervalmin += threshold;
|
||||
if (intervalmin & 0x10000) {
|
||||
addBit();
|
||||
}
|
||||
intervalsize = intervalsize - threshold;
|
||||
new_prob = prob - (prob >> ADJUST_SHIFT);
|
||||
} else {
|
||||
// One
|
||||
intervalsize = threshold;
|
||||
new_prob = prob + (0xffff >> ADJUST_SHIFT) - (prob >> ADJUST_SHIFT);
|
||||
}
|
||||
assert(new_prob > 0);
|
||||
assert(new_prob < 0x10000);
|
||||
contexts[context_index] = new_prob;
|
||||
while (intervalsize < 0x8000) {
|
||||
dest_bit++;
|
||||
intervalsize <<= 1;
|
||||
intervalmin <<= 1;
|
||||
if (intervalmin & 0x10000) {
|
||||
addBit();
|
||||
}
|
||||
}
|
||||
intervalmin &= 0xffff;
|
||||
|
||||
int size_after = (dest_bit << BIT_PRECISION) + sizetable[(intervalsize - 0x8000) >> 8];
|
||||
return size_after - size_before;
|
||||
}
|
||||
|
||||
void reset() {
|
||||
fill(contexts.begin(), contexts.end(), 0x8000);
|
||||
}
|
||||
|
||||
void finish() {
|
||||
int intervalmax = intervalmin + intervalsize;
|
||||
int final_min = 0;
|
||||
int final_size = 0x10000;
|
||||
while (final_min < intervalmin || final_min + final_size >= intervalmax) {
|
||||
if (final_min + final_size < intervalmax) {
|
||||
addBit();
|
||||
final_min += final_size;
|
||||
}
|
||||
dest_bit++;
|
||||
final_size >>= 1;
|
||||
}
|
||||
|
||||
while ((dest_bit - 1) >> 5 >= out.size()) {
|
||||
out.push_back(0);
|
||||
}
|
||||
}
|
||||
|
||||
int sizeInBits() {
|
||||
return dest_bit + 1;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
|
||||
int RangeCoder::sizetable[128];
|
||||
bool RangeCoder::sizetable_init = init_sizetable();
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
A decoder for the range coder.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cmath>
|
||||
#include <algorithm>
|
||||
#include <vector>
|
||||
|
||||
using std::fill;
|
||||
using std::vector;
|
||||
|
||||
#include "Decoder.h"
|
||||
#include "assert.h"
|
||||
|
||||
#ifndef ADJUST_SHIFT
|
||||
#define ADJUST_SHIFT 4
|
||||
#endif
|
||||
|
||||
class CompressedDataReadListener {
|
||||
public:
|
||||
virtual void read(int index) = 0;
|
||||
|
||||
virtual ~CompressedDataReadListener() {}
|
||||
};
|
||||
|
||||
class RangeDecoder : public Decoder {
|
||||
vector<unsigned short> contexts;
|
||||
vector<unsigned>& data;
|
||||
CompressedDataReadListener* listener;
|
||||
int bit_index;
|
||||
unsigned intervalsize;
|
||||
unsigned intervalvalue;
|
||||
unsigned uncertainty;
|
||||
|
||||
int getBit() {
|
||||
int long_index = bit_index >> 5;
|
||||
int bit_in_long = (~bit_index) & 31;
|
||||
if (bit_in_long == 31) {
|
||||
if (listener) listener->read(long_index);
|
||||
}
|
||||
if (bit_index++ >= data.size() * 32) {
|
||||
uncertainty <<= 1;
|
||||
return 0;
|
||||
}
|
||||
int bit = (data[long_index] >> bit_in_long) & 1;
|
||||
return bit;
|
||||
}
|
||||
|
||||
public:
|
||||
RangeDecoder(int n_contexts, vector<unsigned>& data) : data(data) {
|
||||
contexts.resize(n_contexts, 0x8000);
|
||||
bit_index = 0;
|
||||
intervalsize = 1;
|
||||
intervalvalue = 0;
|
||||
uncertainty = 1;
|
||||
listener = NULL;
|
||||
}
|
||||
|
||||
virtual int decode(int context_index) {
|
||||
assert(context_index < contexts.size());
|
||||
unsigned prob = contexts[context_index];
|
||||
while (intervalsize < 0x8000) {
|
||||
intervalsize <<= 1;
|
||||
intervalvalue = (intervalvalue << 1) | getBit();
|
||||
}
|
||||
|
||||
int bit;
|
||||
unsigned new_prob;
|
||||
unsigned threshold = (intervalsize * prob) >> 16;
|
||||
if (intervalvalue >= threshold) {
|
||||
// Zero
|
||||
bit = 0;
|
||||
intervalvalue -= threshold;
|
||||
intervalsize -= threshold;
|
||||
new_prob = prob - (prob >> ADJUST_SHIFT);
|
||||
} else {
|
||||
// One
|
||||
assert(intervalvalue + uncertainty <= threshold);
|
||||
bit = 1;
|
||||
intervalsize = threshold;
|
||||
new_prob = prob + (0xffff >> ADJUST_SHIFT) - (prob >> ADJUST_SHIFT);
|
||||
}
|
||||
assert(new_prob > 0);
|
||||
assert(new_prob < 0x10000);
|
||||
contexts[context_index] = new_prob;
|
||||
|
||||
return bit;
|
||||
}
|
||||
|
||||
void reset() {
|
||||
fill(contexts.begin(), contexts.end(), 0x8000);
|
||||
}
|
||||
|
||||
void setListener(CompressedDataReadListener* listener) {
|
||||
this->listener = listener;
|
||||
}
|
||||
};
|
||||
+361
@@ -0,0 +1,361 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
Main file for the cruncher.
|
||||
|
||||
*/
|
||||
|
||||
//#define SHRINKLER_TITLE ("Shrinkler executable file compressor by Blueberry - version 4.4 (2015-01-18)\n\n")
|
||||
|
||||
#ifndef SHRINKLER_TITLE
|
||||
#define SHRINKLER_TITLE ("Shrinkler executable file compressor by Blueberry - development version (built " __DATE__ " " __TIME__ ")\n\n")
|
||||
#endif
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <string>
|
||||
#include <sys/stat.h>
|
||||
|
||||
using std::string;
|
||||
|
||||
#include "HunkFile.h"
|
||||
#include "DataFile.h"
|
||||
|
||||
void usage() {
|
||||
printf("Usage: Shrinkler <options> <input executable> <output executable>\n");
|
||||
printf("\n");
|
||||
printf("Available options are (default values in parentheses):\n");
|
||||
printf(" -d, --data Treat input as raw data, rather than executable\n");
|
||||
printf(" -h, --hunkmerge Merge hunks of the same memory type\n");
|
||||
printf(" -o, --overlap Overlap compressed and decompressed data to save memory\n");
|
||||
printf(" -m, --mini Use a smaller, but more restricted decrunch header\n");
|
||||
printf(" -i, --iterations Number of iterations for the compression (2)\n");
|
||||
printf(" -l, --length-margin Number of shorter matches considered for each match (2)\n");
|
||||
printf(" -a, --same-length Number of matches of the same length to consider (20)\n");
|
||||
printf(" -e, --effort Perseverance in finding multiple matches (200)\n");
|
||||
printf(" -s, --skip-length Minimum match length to accept greedily (2000)\n");
|
||||
printf(" -r, --references Number of reference edges to keep in memory (100000)\n");
|
||||
printf(" -t, --text Print a text, followed by a newline, before decrunching\n");
|
||||
printf(" -T, --textfile Print the contents of the given file before decrunching\n");
|
||||
printf(" -f, --flash Poke into a register (e.g. DFF180) during decrunching\n");
|
||||
printf(" -p, --no-progress Do not print progress info: no ANSI codes in output\n");
|
||||
printf("\n");
|
||||
exit(0);
|
||||
}
|
||||
|
||||
class Parameter {
|
||||
public:
|
||||
bool seen;
|
||||
|
||||
virtual ~Parameter() {}
|
||||
protected:
|
||||
void parse(const char *form1, const char *form2, const char *arg_kind, int argc, const char *argv[], vector<bool>& consumed) {
|
||||
seen = false;
|
||||
for (int i = 1 ; i < argc ; i++) {
|
||||
if (strcmp(argv[i], form1) == 0 || strcmp(argv[i], form2) == 0) {
|
||||
if (seen) {
|
||||
printf("Error: %s specified multiple times.\n\n", argv[i]);
|
||||
usage();
|
||||
}
|
||||
consumed[i] = true;
|
||||
if (arg_kind) {
|
||||
if (i+1 < argc && !consumed[i+1] && argv[i+1][0] != '-') {
|
||||
seen = parseArg(argv[i], argv[i+1]);
|
||||
}
|
||||
if (!seen) {
|
||||
printf("Error: %s requires a %s argument.\n\n", argv[i], arg_kind);
|
||||
usage();
|
||||
}
|
||||
consumed[i+1] = true;
|
||||
i = i+1;
|
||||
} else {
|
||||
seen = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
virtual bool parseArg(const char *param, const char *arg) = 0;
|
||||
};
|
||||
|
||||
class IntParameter : public Parameter {
|
||||
int min_value;
|
||||
int max_value;
|
||||
public:
|
||||
int value;
|
||||
|
||||
IntParameter(const char *form1, const char *form2, int min_value, int max_value, int default_value,
|
||||
int argc, const char *argv[], vector<bool>& consumed)
|
||||
: min_value(min_value), max_value(max_value), value(default_value)
|
||||
{
|
||||
parse(form1, form2, "numeric", argc, argv, consumed);
|
||||
}
|
||||
|
||||
protected:
|
||||
virtual bool parseArg(const char *param, const char *arg) {
|
||||
char *endptr;
|
||||
value = strtol(arg, &endptr, 10);
|
||||
if (endptr == &arg[strlen(arg)]) {
|
||||
if (value < min_value || value > max_value) {
|
||||
printf("Error: Argument of %s must be between %d and %d.\n\n", param, min_value, max_value);
|
||||
usage();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
class HexParameter : public Parameter {
|
||||
public:
|
||||
unsigned value;
|
||||
|
||||
HexParameter(const char *form1, const char *form2, int default_value,
|
||||
int argc, const char *argv[], vector<bool>& consumed)
|
||||
: value(default_value)
|
||||
{
|
||||
parse(form1, form2, "hexadecimal", argc, argv, consumed);
|
||||
}
|
||||
|
||||
protected:
|
||||
virtual bool parseArg(const char *param, const char *arg) {
|
||||
char *endptr;
|
||||
value = strtol(arg, &endptr, 16);
|
||||
if (endptr == &arg[strlen(arg)]) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
class StringParameter : public Parameter {
|
||||
public:
|
||||
const char *value;
|
||||
|
||||
StringParameter(const char *form1, const char *form2, int argc, const char *argv[], vector<bool>& consumed)
|
||||
: value(NULL)
|
||||
{
|
||||
parse(form1, form2, "string", argc, argv, consumed);
|
||||
}
|
||||
|
||||
protected:
|
||||
virtual bool parseArg(const char *param, const char *arg) {
|
||||
value = arg;
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
class FlagParameter : public Parameter {
|
||||
public:
|
||||
FlagParameter(const char *form1, const char *form2, int argc, const char *argv[], vector<bool>& consumed)
|
||||
{
|
||||
parse(form1, form2, NULL, argc, argv, consumed);
|
||||
}
|
||||
|
||||
protected:
|
||||
virtual bool parseArg(const char *param, const char *arg) {
|
||||
// Not used
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
int main2(int argc, const char *argv[]) {
|
||||
printf(SHRINKLER_TITLE);
|
||||
|
||||
vector<bool> consumed(argc);
|
||||
|
||||
FlagParameter data ("-d", "--data", argc, argv, consumed);
|
||||
FlagParameter hunkmerge ("-h", "--hunkmerge", argc, argv, consumed);
|
||||
FlagParameter overlap ("-o", "--overlap", argc, argv, consumed);
|
||||
FlagParameter mini ("-m", "--mini", argc, argv, consumed);
|
||||
IntParameter iterations ("-i", "--iterations", 1, 9, 2, argc, argv, consumed);
|
||||
IntParameter length_margin ("-l", "--length-margin", 0, 100, 2, argc, argv, consumed);
|
||||
IntParameter same_length ("-a", "--same-length", 1, 100000, 20, argc, argv, consumed);
|
||||
IntParameter effort ("-e", "--effort", 0, 100000, 200, argc, argv, consumed);
|
||||
IntParameter skip_length ("-s", "--skip-length", 2, 100000, 2000, argc, argv, consumed);
|
||||
IntParameter references ("-r", "--references", 1000, 10000000, 100000, argc, argv, consumed);
|
||||
StringParameter text ("-t", "--text", argc, argv, consumed);
|
||||
StringParameter textfile ("-T", "--textfile", argc, argv, consumed);
|
||||
HexParameter flash ("-f", "--flash", 0, argc, argv, consumed);
|
||||
FlagParameter no_progress ("-p", "--no-progress", argc, argv, consumed);
|
||||
|
||||
vector<const char*> files;
|
||||
|
||||
for (int i = 1 ; i < argc ; i++) {
|
||||
if (!consumed[i]) {
|
||||
if (argv[i][0] == '-') {
|
||||
printf("Error: Unknown option %s\n\n", argv[i]);
|
||||
usage();
|
||||
}
|
||||
files.push_back(argv[i]);
|
||||
}
|
||||
}
|
||||
|
||||
if (data.seen && (hunkmerge.seen || overlap.seen || mini.seen || text.seen || textfile.seen || flash.seen)) {
|
||||
printf("Error: The data option cannot be used together with any of the\n");
|
||||
printf("hunkmerge, overlap, mini, text, textfile or flash options.\n\n");
|
||||
usage();
|
||||
}
|
||||
|
||||
if (overlap.seen && mini.seen) {
|
||||
printf("Error: The overlap and mini options cannot be used together.\n\n");
|
||||
usage();
|
||||
}
|
||||
|
||||
if (text.seen && textfile.seen) {
|
||||
printf("Error: The text and textfile options cannot both be specified.\n\n");
|
||||
usage();
|
||||
}
|
||||
|
||||
if (mini.seen && (text.seen || textfile.seen)) {
|
||||
printf("Error: The text and textfile options cannot be used in mini mode.\n\n");
|
||||
usage();
|
||||
}
|
||||
|
||||
if (files.size() == 0) {
|
||||
printf("Error: No input file specified.\n\n");
|
||||
usage();
|
||||
}
|
||||
if (files.size() == 1) {
|
||||
printf("Error: No output file specified.\n\n");
|
||||
usage();
|
||||
}
|
||||
if (files.size() > 2) {
|
||||
printf("Error: Too many files specified.\n\n");
|
||||
usage();
|
||||
}
|
||||
|
||||
const char *infile = files[0];
|
||||
const char *outfile = files[1];
|
||||
|
||||
PackParams params;
|
||||
params.iterations = iterations.value;
|
||||
params.length_margin = length_margin.value;
|
||||
params.skip_length = skip_length.value;
|
||||
params.match_patience = effort.value;
|
||||
params.max_same_length = same_length.value;
|
||||
|
||||
string *decrunch_text_ptr = NULL;
|
||||
string decrunch_text;
|
||||
if (text.seen) {
|
||||
decrunch_text = text.value;
|
||||
decrunch_text.push_back('\n');
|
||||
decrunch_text_ptr = &decrunch_text;
|
||||
} else if (textfile.seen) {
|
||||
FILE *decrunch_text_file = fopen(textfile.value, "r");
|
||||
if (!decrunch_text_file) {
|
||||
printf("Error: Could not open text file %s\n", textfile.value);
|
||||
exit(1);
|
||||
}
|
||||
char c;
|
||||
while ((c = fgetc(decrunch_text_file)) != EOF) {
|
||||
decrunch_text.push_back(c);
|
||||
}
|
||||
fclose(decrunch_text_file);
|
||||
decrunch_text_ptr = &decrunch_text;
|
||||
}
|
||||
|
||||
if (data.seen) {
|
||||
// Data file compression
|
||||
printf("Loading file %s...\n\n", infile);
|
||||
DataFile *orig = new DataFile;
|
||||
orig->load(infile);
|
||||
|
||||
printf("Crunching...\n\n");
|
||||
RefEdgeFactory edge_factory(references.value);
|
||||
DataFile *crunched = orig->crunch(¶ms, &edge_factory, !no_progress.seen);
|
||||
delete orig;
|
||||
printf("References considered:%8d\n", edge_factory.max_edge_count);
|
||||
printf("References discarded:%9d\n\n", edge_factory.max_cleaned_edges);
|
||||
|
||||
printf("Saving file %s...\n\n", outfile);
|
||||
crunched->save(outfile);
|
||||
|
||||
printf("Final file size: %d\n\n", crunched->size());
|
||||
delete crunched;
|
||||
|
||||
if (edge_factory.max_edge_count > references.value) {
|
||||
printf("Note: compression may benefit from a larger reference buffer (-r option).\n\n");
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Executable file compression
|
||||
printf("Loading file %s...\n\n", infile);
|
||||
HunkFile *orig = new HunkFile;
|
||||
orig->load(infile);
|
||||
if (!orig->analyze()) {
|
||||
printf("\nError while analyzing input file!\n\n");
|
||||
delete orig;
|
||||
exit(1);
|
||||
}
|
||||
if (hunkmerge.seen) {
|
||||
printf("Merging hunks...\n\n");
|
||||
HunkFile *merged = orig->merge_hunks(orig->merged_hunklist());
|
||||
delete orig;
|
||||
if (!merged->analyze()) {
|
||||
printf("\nError while analyzing merged file!\n\n");
|
||||
delete merged;
|
||||
internal_error();
|
||||
}
|
||||
orig = merged;
|
||||
}
|
||||
if (mini.seen && !orig->valid_mini()) {
|
||||
printf("Input executable not suitable for mini crunching.\n"
|
||||
"Must contain only one non-empty hunk and no relocations,\n"
|
||||
"and the final file size must be less than 24k.\n\n");
|
||||
delete orig;
|
||||
exit(1);
|
||||
}
|
||||
int orig_mem = orig->memory_usage(true);
|
||||
printf("Crunching...\n\n");
|
||||
RefEdgeFactory edge_factory(references.value);
|
||||
HunkFile *crunched = orig->crunch(¶ms, overlap.seen, mini.seen, decrunch_text_ptr, flash.value, &edge_factory, !no_progress.seen);
|
||||
delete orig;
|
||||
printf("References considered:%8d\n", edge_factory.max_edge_count);
|
||||
printf("References discarded:%9d\n\n", edge_factory.max_cleaned_edges);
|
||||
if (!crunched->analyze()) {
|
||||
printf("\nError while analyzing crunched file!\n\n");
|
||||
delete crunched;
|
||||
internal_error();
|
||||
}
|
||||
int crunched_mem_during = crunched->memory_usage(true);
|
||||
int crunched_mem_after = crunched->memory_usage(mini.seen || overlap.seen);
|
||||
|
||||
printf("Memory overhead during decrunching: %9d\n", crunched_mem_during - orig_mem);
|
||||
printf("Memory overhead after decrunching: %9d\n\n", crunched_mem_after - orig_mem);
|
||||
|
||||
printf("Saving file %s...\n\n", outfile);
|
||||
crunched->save(outfile);
|
||||
#ifdef S_IRWXU // Is the POSIX file permission API available?
|
||||
chmod(outfile, 0755); // Mark file executable
|
||||
#endif
|
||||
|
||||
printf("Final file size: %d\n\n", crunched->size());
|
||||
delete crunched;
|
||||
|
||||
if (edge_factory.max_edge_count > references.value) {
|
||||
printf("Note: compression may benefit from a larger reference buffer (-r option).\n\n");
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, const char *argv[]) {
|
||||
try {
|
||||
return main2(argc, argv);
|
||||
} catch (std::bad_alloc& e) {
|
||||
fflush(stdout);
|
||||
fprintf(stderr,
|
||||
"\n\nShrinkler ran out of memory.\n\n"
|
||||
"Some things you can try:\n"
|
||||
" - Free up some memory\n"
|
||||
" - Run it on a machine with more memory\n"
|
||||
" - Reduce the size of the reference buffer (-r option)\n"
|
||||
" - Split up your biggest hunk into smaller ones\n\n");
|
||||
fflush(stderr);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
; Copyright 1999-2015 Aske Simon Christensen.
|
||||
;
|
||||
; The code herein is free to use, in whole or in part,
|
||||
; modified or as is, for any legal purpose.
|
||||
;
|
||||
; No warranties of any kind are given as to its behavior
|
||||
; or suitability.
|
||||
|
||||
|
||||
INIT_ONE_PROB = $8000
|
||||
ADJUST_SHIFT = 4
|
||||
SINGLE_BIT_CONTEXTS = 1
|
||||
NUM_CONTEXTS = 1536
|
||||
|
||||
|
||||
; Decompress Shrinkler-compressed data produced with the --data option.
|
||||
;
|
||||
; A0 = Compressed data
|
||||
; A1 = Decompressed data destination
|
||||
; A2 = Progress callback, can be zero if no callback is desired.
|
||||
; Callback will be called continuously with
|
||||
; D0 = Number of bytes decompressed so far
|
||||
; A0 = Callback argument
|
||||
; A3 = Callback argument
|
||||
;
|
||||
; Uses 3 kilobytes of space on the stack.
|
||||
; Preserves D2-D7/A2-A6 and assumes callback does the same.
|
||||
;
|
||||
; Decompression code may read one longword beyond compressed data.
|
||||
; The contents of this longword does not matter.
|
||||
|
||||
ShrinklerDecompress:
|
||||
movem.l d2-d7/a4-a6,-(a7)
|
||||
|
||||
move.l a0,a4
|
||||
move.l a1,a5
|
||||
move.l a1,a6
|
||||
|
||||
; Init range decoder state
|
||||
moveq.l #0,d2
|
||||
moveq.l #1,d3
|
||||
moveq.l #1,d4
|
||||
ror.l #1,d4
|
||||
|
||||
; Init probabilities
|
||||
move.l #NUM_CONTEXTS,d6
|
||||
.init: move.w #INIT_ONE_PROB,-(a7)
|
||||
subq.w #1,d6
|
||||
bne.b .init
|
||||
|
||||
; D6 = 0
|
||||
.lit:
|
||||
; Literal
|
||||
addq.b #1,d6
|
||||
.getlit:
|
||||
bsr.b GetBit
|
||||
addx.b d6,d6
|
||||
bcc.b .getlit
|
||||
move.b d6,(a5)+
|
||||
bsr.b ReportProgress
|
||||
.switch:
|
||||
; After literal
|
||||
bsr.b GetKind
|
||||
bcc.b .lit
|
||||
; Reference
|
||||
moveq.l #-1,d6
|
||||
bsr.b GetBit
|
||||
bcc.b .readoffset
|
||||
.readlength:
|
||||
moveq.l #4,d6
|
||||
bsr.b GetNumber
|
||||
.copyloop:
|
||||
move.b (a5,d5.l),(a5)+
|
||||
subq.l #1,d7
|
||||
bne.b .copyloop
|
||||
bsr.b ReportProgress
|
||||
; After reference
|
||||
bsr.b GetKind
|
||||
bcc.b .lit
|
||||
.readoffset:
|
||||
moveq.l #3,d6
|
||||
bsr.b GetNumber
|
||||
moveq.l #2,d5
|
||||
sub.l d7,d5
|
||||
bne.b .readlength
|
||||
|
||||
lea.l NUM_CONTEXTS*2(a7),a7
|
||||
movem.l (a7)+,d2-d7/a4-a6
|
||||
rts
|
||||
|
||||
ReportProgress:
|
||||
move.l a2,d0
|
||||
beq.b .nocallback
|
||||
move.l a5,d0
|
||||
sub.l a6,d0
|
||||
move.l a3,a0
|
||||
jsr (a2)
|
||||
.nocallback:
|
||||
rts
|
||||
|
||||
GetKind:
|
||||
; Use parity as context
|
||||
move.l a5,d1
|
||||
moveq.l #1,d6
|
||||
and.l d1,d6
|
||||
lsl.w #8,d6
|
||||
bra.b GetBit
|
||||
|
||||
GetNumber:
|
||||
; D6 = Number context
|
||||
|
||||
; Out: Number in D7
|
||||
lsl.w #8,d6
|
||||
.numberloop:
|
||||
addq.b #2,d6
|
||||
bsr.b GetBit
|
||||
bcs.b .numberloop
|
||||
moveq.l #1,d7
|
||||
subq.b #1,d6
|
||||
.bitsloop:
|
||||
bsr.b GetBit
|
||||
addx.l d7,d7
|
||||
subq.b #2,d6
|
||||
bcc.b .bitsloop
|
||||
rts
|
||||
|
||||
; D6 = Bit context
|
||||
|
||||
; D2 = Range value
|
||||
; D3 = Interval size
|
||||
; D4 = Input bit buffer
|
||||
|
||||
; Out: Bit in C and X
|
||||
|
||||
readbit:
|
||||
add.l d4,d4
|
||||
bne.b nonewword
|
||||
move.l (a4)+,d4
|
||||
addx.l d4,d4
|
||||
nonewword:
|
||||
addx.w d2,d2
|
||||
add.w d3,d3
|
||||
GetBit:
|
||||
tst.w d3
|
||||
bpl.b readbit
|
||||
|
||||
lea.l 4+SINGLE_BIT_CONTEXTS*2(a7,d6.l),a1
|
||||
add.l d6,a1
|
||||
move.w (a1),d1
|
||||
; D1 = One prob
|
||||
|
||||
lsr.w #ADJUST_SHIFT,d1
|
||||
sub.w d1,(a1)
|
||||
add.w (a1),d1
|
||||
|
||||
mulu.w d3,d1
|
||||
swap.w d1
|
||||
|
||||
sub.w d1,d2
|
||||
blo.b .one
|
||||
.zero:
|
||||
; oneprob = oneprob * (1 - adjust) = oneprob - oneprob * adjust
|
||||
sub.w d1,d3
|
||||
; 0 in C and X
|
||||
rts
|
||||
.one:
|
||||
; onebrob = 1 - (1 - oneprob) * (1 - adjust) = oneprob - oneprob * adjust + adjust
|
||||
add.w #$ffff>>ADJUST_SHIFT,(a1)
|
||||
move.w d1,d3
|
||||
add.w d1,d2
|
||||
; 1 in C and X
|
||||
rts
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
A dummy entropy coder which estimates the size of coded symbols based on
|
||||
counts from a CountingCoder.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
|
||||
using std::vector;
|
||||
|
||||
#include "CountingCoder.h"
|
||||
|
||||
struct ContextSizes {
|
||||
unsigned short sizes[2];
|
||||
};
|
||||
|
||||
class SizeMeasuringCoder : public Coder {
|
||||
static const int MIN_SIZE = 2;
|
||||
static const int MAX_SIZE = 12 << BIT_PRECISION;
|
||||
|
||||
vector<ContextSizes> context_sizes;
|
||||
|
||||
int sizeForCount(int count, int total) {
|
||||
int size = (int) floor(0.5 + log(total / (double) count) / log(2.0) * (1 << BIT_PRECISION));
|
||||
if (size < MIN_SIZE) size = MIN_SIZE;
|
||||
if (size > MAX_SIZE) size = MAX_SIZE;
|
||||
return size;
|
||||
}
|
||||
|
||||
public:
|
||||
SizeMeasuringCoder(int n_contexts) {
|
||||
struct ContextSizes default_sizes = { { 1 << BIT_PRECISION, 1 << BIT_PRECISION } };
|
||||
context_sizes.resize(n_contexts, default_sizes);
|
||||
setCacheable(true);
|
||||
}
|
||||
|
||||
SizeMeasuringCoder(CountingCoder *counting_coder) {
|
||||
context_sizes.resize(counting_coder->context_counts.size());
|
||||
for (int i = 0 ; i < counting_coder->context_counts.size() ; i++) {
|
||||
struct ContextSizes s;
|
||||
struct ContextCounts c = counting_coder->context_counts[i];
|
||||
int count0 = 1 + c.counts[0];
|
||||
int count1 = 1 + c.counts[1];
|
||||
int sum = count0 + count1;
|
||||
s.sizes[0] = sizeForCount(count0, sum);
|
||||
s.sizes[1] = sizeForCount(count1, sum);
|
||||
context_sizes[i] = s;
|
||||
}
|
||||
setCacheable(true);
|
||||
}
|
||||
|
||||
virtual int code(int context_index, int bit) {
|
||||
return context_sizes[context_index].sizes[bit];
|
||||
}
|
||||
};
|
||||
Vendored
+37
@@ -0,0 +1,37 @@
|
||||
// Copyright 1999-2015 Aske Simon Christensen. See LICENSE.txt for usage terms.
|
||||
|
||||
/*
|
||||
|
||||
An assert function which contains a breakpoint, for ease of debugging.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
void internal_error() {
|
||||
fflush(stdout);
|
||||
fprintf(stderr,
|
||||
"\n\nShrinkler has encountered an internal error.\n"
|
||||
"Please send a bug report to blueberry@loonies.dk,\n"
|
||||
"providing the file you tried to compress.\n"
|
||||
"\n"
|
||||
"Thanks, and apologies for the inconvenience.\n\n");
|
||||
fflush(stderr);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
#ifndef NDEBUG
|
||||
#include <stdio.h>
|
||||
static void _assert_func(const char *file, int line, const char *exp) {
|
||||
fflush(stdout);
|
||||
fprintf(stderr, "\n\nassertion \"%s\" failed: file \"%s\", line %d\n", exp, file, line);
|
||||
fflush(stderr);
|
||||
#ifdef DEBUG
|
||||
__asm volatile ("int3;");
|
||||
#endif
|
||||
internal_error();
|
||||
}
|
||||
#undef assert
|
||||
#define assert(__e) ((__e) ? (void)0 : _assert_func (__FILE__, __LINE__, #__e))
|
||||
#endif
|
||||
|
||||
Vendored
+95
@@ -0,0 +1,95 @@
|
||||
#ifndef DOS_DOSHUNKS_H
|
||||
#define DOS_DOSHUNKS_H
|
||||
/*
|
||||
** $VER: doshunks.h 36.9 (2.6.92)
|
||||
** Includes Release 40.13
|
||||
**
|
||||
** Hunk definitions for object and load modules.
|
||||
**
|
||||
** (C) Copyright 1989-1993 Commodore-Amiga, Inc.
|
||||
** All Rights Reserved
|
||||
*/
|
||||
|
||||
/* hunk types */
|
||||
#define HUNK_UNIT 999
|
||||
#define HUNK_NAME 1000
|
||||
#define HUNK_CODE 1001
|
||||
#define HUNK_DATA 1002
|
||||
#define HUNK_BSS 1003
|
||||
#define HUNK_RELOC32 1004
|
||||
#define HUNK_ABSRELOC32 HUNK_RELOC32
|
||||
#define HUNK_RELOC16 1005
|
||||
#define HUNK_RELRELOC16 HUNK_RELOC16
|
||||
#define HUNK_RELOC8 1006
|
||||
#define HUNK_RELRELOC8 HUNK_RELOC8
|
||||
#define HUNK_EXT 1007
|
||||
#define HUNK_SYMBOL 1008
|
||||
#define HUNK_DEBUG 1009
|
||||
#define HUNK_END 1010
|
||||
#define HUNK_HEADER 1011
|
||||
|
||||
#define HUNK_OVERLAY 1013
|
||||
#define HUNK_BREAK 1014
|
||||
|
||||
#define HUNK_DREL32 1015
|
||||
#define HUNK_DREL16 1016
|
||||
#define HUNK_DREL8 1017
|
||||
|
||||
#define HUNK_LIB 1018
|
||||
#define HUNK_INDEX 1019
|
||||
|
||||
/*
|
||||
* Note: V37 LoadSeg uses 1015 (HUNK_DREL32) by mistake. This will continue
|
||||
* to be supported in future versions, since HUNK_DREL32 is illegal in load files
|
||||
* anyways. Future versions will support both 1015 and 1020, though anything
|
||||
* that should be usable under V37 should use 1015.
|
||||
*/
|
||||
#define HUNK_RELOC32SHORT 1020
|
||||
|
||||
/* see ext_xxx below. New for V39 (note that LoadSeg only handles RELRELOC32).*/
|
||||
#define HUNK_RELRELOC32 1021
|
||||
#define HUNK_ABSRELOC16 1022
|
||||
|
||||
/*
|
||||
* Any hunks that have the HUNKB_ADVISORY bit set will be ignored if they
|
||||
* aren't understood. When ignored, they're treated like HUNK_DEBUG hunks.
|
||||
* NOTE: this handling of HUNKB_ADVISORY started as of V39 dos.library! If
|
||||
* lading such executables is attempted under <V39 dos, it will fail with a
|
||||
* bad hunk type.
|
||||
*/
|
||||
#define HUNKB_ADVISORY 29
|
||||
#define HUNKB_CHIP 30
|
||||
#define HUNKB_FAST 31
|
||||
#define HUNKF_ADVISORY (1L<<29)
|
||||
#define HUNKF_CHIP (1L<<30)
|
||||
#define HUNKF_FAST (1L<<31)
|
||||
|
||||
|
||||
/* hunk_ext sub-types */
|
||||
#define EXT_SYMB 0 /* symbol table */
|
||||
#define EXT_DEF 1 /* relocatable definition */
|
||||
#define EXT_ABS 2 /* Absolute definition */
|
||||
#define EXT_RES 3 /* no longer supported */
|
||||
#define EXT_REF32 129 /* 32 bit absolute reference to symbol */
|
||||
#define EXT_ABSREF32 EXT_REF32
|
||||
#define EXT_COMMON 130 /* 32 bit absolute reference to COMMON block */
|
||||
#define EXT_ABSCOMMON EXT_COMMON
|
||||
#define EXT_REF16 131 /* 16 bit PC-relative reference to symbol */
|
||||
#define EXT_RELREF16 EXT_REF16
|
||||
#define EXT_REF8 132 /* 8 bit PC-relative reference to symbol */
|
||||
#define EXT_RELREF8 EXT_REF8
|
||||
#define EXT_DEXT32 133 /* 32 bit data relative reference */
|
||||
#define EXT_DEXT16 134 /* 16 bit data relative reference */
|
||||
#define EXT_DEXT8 135 /* 8 bit data relative reference */
|
||||
|
||||
/* These are to support some of the '020 and up modes that are rarely used */
|
||||
#define EXT_RELREF32 136 /* 32 bit PC-relative reference to symbol */
|
||||
#define EXT_RELCOMMON 137 /* 32 bit PC-relative reference to COMMON block */
|
||||
|
||||
/* for completeness... All 680x0's support this */
|
||||
#define EXT_ABSREF16 138 /* 16 bit absolute reference to symbol */
|
||||
|
||||
/* this only exists on '020's and above, in the (d8,An,Xn) address mode */
|
||||
#define EXT_ABSREF8 139 /* 8 bit absolute reference to symbol */
|
||||
|
||||
#endif /* DOS_DOSHUNKS_H */
|
||||
Binary file not shown.
Binary file not shown.
Reference in New Issue
Block a user