diff --git a/examples/push.js b/examples/push.js new file mode 100644 index 0000000..578511e --- /dev/null +++ b/examples/push.js @@ -0,0 +1,26 @@ +var platform = require('git-node-platform'); +var jsGit = require('../.')(platform); +var gitRemote = require('git-net')(platform); +var fsDb = require('git-fs-db')(platform); +var fs = platform.fs; +var basename = require('path').basename; + +// Create a remote repo +var url = process.argv[2] || "git@github.com:creationix/conquest.git"; +var remote = gitRemote(url); + +// Create a local repo +var path = process.argv[3] || basename(remote.pathname); +var repo = jsGit(fsDb(fs(path))); + +console.log("Pushing %s to %s", path, url); + +var opts = { + onProgress: function (progress) { + process.stderr.write(progress); + } +}; +repo.send(remote, opts, function (err) { + if (err) throw err; + console.log("Done"); +}); diff --git a/js-git.js b/js-git.js index 1508c8b..be7d548 100644 --- a/js-git.js +++ b/js-git.js @@ -75,7 +75,7 @@ function newRepo(db, workDir) { // Network Protocols repo.fetch = fetch; - repo.push = push; + repo.send = send; return repo; @@ -539,8 +539,8 @@ function newRepo(db, workDir) { str += "\nparent " + parents[i]; } str += "\nauthor " + encodePerson(commit.author) + - "\ncommitter " + encodePerson(commit.committer || commit.author) + - "\n\n" + commit.message; + "\ncommitter " + encodePerson(commit.committer || commit.author) + + "\n\n" + commit.message; return bops.from(str); } @@ -840,8 +840,9 @@ function newRepo(db, workDir) { } } - function push() { - throw new Error("TODO: Implement repo.fetch"); + function send(remote, opts, callback) { + + throw new Error("TODO: Implement repo.send"); } function unpack(packStream, opts, callback) { diff --git a/lib/apply-delta.js b/lib/apply-delta.js new file mode 100644 index 0000000..01259c1 --- /dev/null +++ b/lib/apply-delta.js @@ -0,0 +1,94 @@ +module.exports = function (platform) { + var binary = platform.bops + , Decoder = require('varint/decode.js') + , vi = new Decoder + + // we use writeUint[8|32][LE|BE] instead of indexing + // into buffers so that we get buffer-browserify compat. + var OFFSET_BUFFER = binary.create(4) + , LENGTH_BUFFER = binary.create(4) + + return apply_delta; + function apply_delta(delta, target) { + var base_size_info = {size: null, buffer: null} + , resized_size_info = {size: null, buffer: null} + , output_buffer + , out_idx + , command + , len + , idx + + delta_header(delta, base_size_info) + delta_header(base_size_info.buffer, resized_size_info) + + delta = resized_size_info.buffer + + idx = + out_idx = 0 + output_buffer = binary.create(resized_size_info.size) + + len = delta.length + + while(idx < len) { + command = delta[idx++] + command & 0x80 ? copy() : insert() + } + + return output_buffer + + function copy() { + binary.writeUInt32LE(OFFSET_BUFFER, 0, 0) + binary.writeUInt32LE(LENGTH_BUFFER, 0, 0) + + var check = 1 + , length + , offset + + for(var x = 0; x < 4; ++x) { + if(command & check) { + OFFSET_BUFFER[3 - x] = delta[idx++] + } + check <<= 1 + } + + for(var x = 0; x < 3; ++x) { + if(command & check) { + LENGTH_BUFFER[3 - x] = delta[idx++] + } + check <<= 1 + } + LENGTH_BUFFER[0] = 0 + + length = binary.readUInt32BE(LENGTH_BUFFER, 0) || 0x10000 + offset = binary.readUInt32BE(OFFSET_BUFFER, 0) + + binary.copy(target, output_buffer, out_idx, offset, offset + length) + out_idx += length + } + + function insert() { + binary.copy(delta, output_buffer, out_idx, idx, command + idx) + idx += command + out_idx += command + } + } + + function delta_header(buf, output) { + var done = false + , idx = 0 + , size = 0 + + vi.ondata = function(s) { + size = s + done = true + } + + do { + vi.write(buf[idx++]) + } while(!done) + + output.size = size + output.buffer = binary.subarray(buf, idx) + } + +} \ No newline at end of file diff --git a/lib/binary.js b/lib/binary.js new file mode 100644 index 0000000..eed5608 --- /dev/null +++ b/lib/binary.js @@ -0,0 +1,109 @@ +var create, subarray, isBinary; +if (typeof Buffer === "undefined") { + create = function (length) { + return new Uint8Array(length|0); + }; + isBinary = function (binary) { + return binary instanceof Uint8Array; + }; + subarray = function (binary, start, end) { + return binary.subarray(start, end); + }; +} +else { + create = function (length) { + return new Buffer(length|0); + }; + isBinary = Buffer.isBuffer; + subarray = function (buffer, start, end) { + return buffer.slice(start, end); + }; +} + +module.exports = { + create: create, + build: build, + is: isBinary, + subarray: subarray, + copy: copy, + read: read, + write: write, +}; + +// Build a binary value from parts +// parts can be: +// integer - empty space +// string - binary string +// array-like value of bytes (including binary buffers) +var slice = Function.prototype.call.bind(Array.prototype.slice); +function build() { + var parts = slice(arguments); + var i, part, binary; + var length = parts.length|0; + var total = 0; + var offset = 0; + + // Measure the total length needed and encode strings. + for (i = 0; i < length; i++) { + part = parts[i]; + if (part === (part|0)) total += part; + else if (part && (part.length === (part.length|0))) total += part.length; + else throw new Error("Invalid part: " + part + " at offset " + i); + } + + // Copy the items into the new binary buffer. + binary = create(total); + for (i = 0; i < length; i++) { + part = parts[i]; + if (part === (part|0)) offset += part; + else { + if (typeof part === "string") write(binary, part, offset); + else copy(part, binary, offset); + offset += part.length; + } + } + + return binary; +} + +// Write source binary into target binary at optional offset +function copy(source, target, offset) { + if (typeof offset !== "number") offset = 0; + var length = source.length; + var i; + if (source.buffer === target.buffer) { + // Copy source in case source and target overlap + // TODO: look deeper to see if they actually overlap and how. + var temp = new Array(length); + for (i = 0; i < length; i++) { + temp[i] = source[i]; + } + source = temp; + } + for (i = 0; i < length; i++) { + target[offset + i] = source[i]; + } + return target; +} + +// Read a string from the binary value +function read(binary, offset, length) { + if (offset !== (offset|0)) offset = 0; + if (length !== (length|0)) length = binary.length; + var end = offset + length; + var string= ""; + for (var i = offset; i < end; i++) { + string += String.fromCharCode(binary[i]); + } + return string; +} + +// Write a string to the binary value +function write(binary, string, offset) { + if (offset !== (offset|0)) offset = 0; + var length = string.length; + for (var i = 0; i < length; i ++) { + binary[offset + i] = string.charCodeAt(i) & 0xff; + } + return binary; +} diff --git a/lib/decode.js b/lib/decode.js new file mode 100644 index 0000000..d53424f --- /dev/null +++ b/lib/decode.js @@ -0,0 +1,190 @@ + +var types = { + "1": "commit", + "2": "tree", + "3": "blob", + "4": "tag", + "6": "ofs-delta", + "7": "ref-delta" +}; + +module.exports = function (platform) { + var inflate = require('./inflate.js')(platform); + var bops = platform.bops; + var sha1 = platform.sha1; + + return function (emit) { + + var state = $pack; + var sha1sum = sha1(); + var inf = inflate(); + + var offset = 0; + var position = 0; + var version = 0x4b434150; // PACK reversed + var num = 0; + var type = 0; + var length = 0; + var ref = null; + var checksum = ""; + var start = 0; + var parts = []; + + + return function (chunk) { + if (chunk === undefined) { + if (num || checksum.length < 40) throw new Error("Unexpected end of input stream"); + return emit(); + } + + for (var i = 0, l = chunk.length; i < l; i++) { + // console.log([state, i, chunk[i].toString(16)]); + if (!state) throw new Error("Unexpected extra bytes: " + bops.subarray(chunk, i)); + state = state(chunk[i], i, chunk); + position++; + } + if (!state) return; + if (state !== $checksum) sha1sum.update(chunk); + var buff = inf.flush(); + if (buff.length) { + parts.push(buff); + } + }; + + // The first four bytes in a packfile are the bytes 'PACK' + function $pack(byte) { + if ((version & 0xff) === byte) { + version >>>= 8; + return version ? $pack : $version; + } + throw new Error("Invalid packfile header"); + } + + // The version is stored as an unsigned 32 integer in network byte order. + // It must be version 2 or 3. + function $version(byte) { + version = (version << 8) | byte; + if (++offset < 4) return $version; + if (version >= 2 && version <= 3) { + offset = 0; + return $num; + } + throw new Error("Invalid version number " + num); + } + + // The number of objects in this packfile is also stored as an unsigned 32 bit int. + function $num(byte) { + num = (num << 8) | byte; + if (++offset < 4) return $num; + offset = 0; + emit({version: version, num: num}); + return $header; + } + + // n-byte type and length (3-bit type, (n-1)*7+4-bit length) + // CTTTSSSS + // C is continue bit, TTT is type, S+ is length + function $header(byte) { + if (start === 0) start = position; + type = byte >> 4 & 0x07; + length = byte & 0x0f; + if (byte & 0x80) { + offset = 4; + return $header2; + } + return afterHeader(); + } + + // Second state in the same header parsing. + // CSSSSSSS* + function $header2(byte) { + length |= (byte & 0x7f) << offset; + if (byte & 0x80) { + offset += 7; + return $header2; + } + return afterHeader(); + } + + // Common helper for finishing tiny and normal headers. + function afterHeader() { + offset = 0; + if (type === 6) { + ref = 0; + return $ofsDelta; + } + if (type === 7) { + ref = ""; + return $refDelta; + } + return $body; + } + + // Big-endian modified base 128 number encoded ref offset + function $ofsDelta(byte) { + ref = byte & 0x7f; + if (byte & 0x80) return $ofsDelta2; + return $body; + } + + function $ofsDelta2(byte) { + ref = ((ref + 1) << 7) | (byte & 0x7f); + if (byte & 0x80) return $ofsDelta2; + return $body; + } + + // 20 byte raw sha1 hash for ref + function $refDelta(byte) { + ref += toHex(byte); + if (++offset < 20) return $refDelta; + return $body; + } + + // Common helper for generating 2-character hex numbers + function toHex(num) { + return num < 0x10 ? "0" + num.toString(16) : num.toString(16); + } + + // Common helper for emitting all three object shapes + function emitObject() { + var item = { + type: types[type], + size: length, + body: bops.join(parts), + offset: start + }; + if (ref) item.ref = ref; + parts.length = 0; + start = 0; + offset = 0; + type = 0; + length = 0; + ref = null; + emit(item); + } + + // Feed the deflated code to the inflate engine + function $body(byte, i, chunk) { + if (inf.write(byte)) return $body; + var buf = inf.flush(); + inf.recycle(); + if (buf.length) { + parts.push(buf); + } + emitObject(); + // If this was all the objects, start calculating the sha1sum + if (--num) return $header; + sha1sum.update(bops.subarray(chunk, 0, i + 1)); + return $checksum; + } + + // 20 byte checksum + function $checksum(byte) { + checksum += toHex(byte); + if (++offset < 20) return $checksum; + var actual = sha1sum.digest(); + if (checksum !== actual) throw new Error("Checksum mismatch: " + actual + " != " + checksum); + } + + }; +}; diff --git a/lib/inflate.js b/lib/inflate.js new file mode 100644 index 0000000..196b8b3 --- /dev/null +++ b/lib/inflate.js @@ -0,0 +1,56 @@ +module.exports = function (platform) { + + var inflate = require('./min.js')(platform); + var bops = platform.bops; + + // Wrapper for proposed new API to inflate: + // + // var inf = inflate(); + // inf.write(byte) -> more - Write a byte to inflate's state-machine. + // Returns true if more data is expected. + // inf.recycle() - Reset the internal state machine. + // inf.flush() -> data - Flush the output as a binary buffer. + // + // This is quite slow, but could be made fast if baked into inflate itself. + return function () { + var push = inflate(onEmit, onUnused); + var more = true; + var chunks = []; + var b = bops.create(1); + + return { write: write, recycle: recycle, flush: flush }; + + function write(byte) { + b[0] = byte; + push(null, b); + return more; + } + + function recycle() { + push.recycle(); + more = true; + } + + function flush() { + var buffer = bops.join(chunks); + chunks.length = 0; + return buffer; + } + + function onEmit(err, item) { + if (err) throw err; + if (item === undefined) { + // console.log("onEnd"); + more = false; + return; + } + chunks.push(item); + } + + function onUnused(chunks) { + // console.log("onUnused", chunks); + more = false; + } + }; + +}; \ No newline at end of file diff --git a/lib/min.js b/lib/min.js new file mode 100644 index 0000000..481b48c --- /dev/null +++ b/lib/min.js @@ -0,0 +1,808 @@ +module.exports = function (platform) { + var binary = platform.bops; + + var MAXBITS = 15 + , MAXLCODES = 286 + , MAXDCODES = 30 + , MAXCODES = (MAXLCODES+MAXDCODES) + , FIXLCODES = 288 + + var lens = [ + 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, + 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258 + ] + + var lext = [ + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, + 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0 + ] + + var dists = [ + 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, + 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, + 8193, 12289, 16385, 24577 + ] + + var dext = [ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, + 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, + 12, 12, 13, 13 + ] + + var order = [ + 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 + ] + + var WINDOW = 32768 + , WINDOW_MINUS_ONE = WINDOW - 1 + + function inflate(emit, on_unused) { + var output = new Uint8Array(WINDOW) + , need_input = false + , buffer_offset = 0 + , bytes_read = 0 + , output_idx = 0 + , ended = false + , state = null + , states = [] + , buffer = [] + , got = 0 + + // buffer up to 128k "output one" bytes + var OUTPUT_ONE_LENGTH = 131070 + , output_one_offs = OUTPUT_ONE_LENGTH + , output_one_buf + + var bitbuf = 0 + , bitcnt = 0 + , is_final = false + , fixed_codes + + var adler_s1 = 1 + , adler_s2 = 0 + + onread.recycle = function recycle() { + var out + buffer.length = 0 + buffer_offset = 0 + output_idx = 0 + bitbuf = 0 + bitcnt = 0 + states.length = 0 + is_final = false + need_input = false + bytes_read = 0 + output_idx = 0 + ended = false + got = 0 + adler_s1 = 1 + adler_s2 = 0 + output_one_offs = 0 + become(noop, {}, noop) + start_stream_header() + // return stream + } + + var bytes_need = 0 + , bytes_value = [] + + var bits_need = 0 + , bits_value = [] + + var codes_distcode = null + , codes_lencode = null + , codes_len = 0 + , codes_dist = 0 + , codes_symbol = 0 + + var dynamic_distcode = {symbol: [], count: []} + , dynamic_lencode = {symbol: [], count: []} + , dynamic_lengths = [] + , dynamic_nlen = 0 + , dynamic_ndist = 0 + , dynamic_ncode = 0 + , dynamic_index = 0 + , dynamic_symbol = 0 + , dynamic_len = 0 + + var decode_huffman = null + , decode_len = 0 + , decode_code = 0 + , decode_first = 0 + , decode_count = 0 + , decode_index = 0 + + var last = null + + become(noop, {}, noop) + start_stream_header() + + return onread + + function onread(err, buf) { + if(buf === undefined) { + return emit(err) + } + + return write(buf) + } + + function noop() { + + } + + function call_header() { + } + + function call_bytes(need) { + bytes_value.length = 0 + bytes_need = need + } + + function call_bits(need) { + bits_value = 0 + bits_need = need + } + + function call_codes(distcode, lencode) { + codes_len = + codes_dist = + codes_symbol = 0 + codes_distcode = distcode + codes_lencode = lencode + } + + function call_dynamic() { + dynamic_distcode.symbol.length = + dynamic_distcode.count.length = + dynamic_lencode.symbol.length = + dynamic_lencode.count.length = + dynamic_lengths.length = 0 + dynamic_nlen = 0 + dynamic_ndist = 0 + dynamic_ncode = 0 + dynamic_index = 0 + dynamic_symbol = 0 + dynamic_len = 0 + } + + function call_decode(h) { + decode_huffman = h + decode_len = 1 + decode_first = + decode_index = + decode_code = 0 + } + + function write(buf) { + buffer.push(buf) + got += buf.length + if(!ended) { + execute() + } + } + + function execute() { + do { + states[0].current() + } while(!need_input && !ended) + + var needed = need_input + need_input = false + } + + function start_stream_header() { + become(bytes, call_bytes(2), got_stream_header) + } + + function got_stream_header() { + var cmf = last[0] + , flg = last[1] + + + if((cmf << 8 | flg) % 31 !== 0) { + emit(new Error( + 'failed header check' + )) + return + } + + + + + if(flg & 32) { + return become(bytes, call_bytes(4), on_got_fdict) + } + return become(bits, call_bits(1), on_got_is_final) + } + + + + + function on_got_fdict() { + return become(bits, call_bits(1), on_got_is_final) + } + + + + + + + + + function on_got_is_final() { + is_final = last + become(bits, call_bits(2), on_got_type) + } + + + + + + + + + + + + + function on_got_type() { + if(last === 0) { + become(bytes, call_bytes(4), on_got_len_nlen) + return + } + + if(last === 1) { + // `fixed` and `dynamic` blocks both eventually delegate + // to the "codes" state -- which reads bits of input, throws + // them into a huffman tree, and produces "symbols" of output. + fixed_codes = fixed_codes || build_fixed() + become(start_codes, call_codes( + fixed_codes.distcode + , fixed_codes.lencode + ), done_with_codes) + return + } + + become(start_dynamic, call_dynamic(), done_with_codes) + return + } + + + + + function on_got_len_nlen() { + var want = last[0] | (last[1] << 8) + , nlen = last[2] | (last[3] << 8) + + if((~nlen & 0xFFFF) !== want) { + emit(new Error( + 'failed len / nlen check' + )) + } + + if(!want) { + become(bits, call_bits(1), on_got_is_final) + return + } + become(bytes, call_bytes(want), on_got_stored) + } + + + + + function on_got_stored() { + output_many(last) + if(is_final) { + become(bytes, call_bytes(4), on_got_adler) + return + } + become(bits, call_bits(1), on_got_is_final) + } + + + + + + + function start_dynamic() { + become(bits, call_bits(5), on_got_nlen) + } + + function on_got_nlen() { + dynamic_nlen = last + 257 + become(bits, call_bits(5), on_got_ndist) + } + + function on_got_ndist() { + dynamic_ndist = last + 1 + become(bits, call_bits(4), on_got_ncode) + } + + function on_got_ncode() { + dynamic_ncode = last + 4 + if(dynamic_nlen > MAXLCODES || dynamic_ndist > MAXDCODES) { + emit(new Error('bad counts')) + return + } + + become(bits, call_bits(3), on_got_lengths_part) + } + + function on_got_lengths_part() { + dynamic_lengths[order[dynamic_index]] = last + + ++dynamic_index + if(dynamic_index === dynamic_ncode) { + for(; dynamic_index < 19; ++dynamic_index) { + dynamic_lengths[order[dynamic_index]] = 0 + } + + // temporarily construct the `lencode` using the + // lengths we've read. we'll actually be using the + // symbols produced by throwing bits into the huffman + // tree to constuct the `lencode` and `distcode` huffman + // trees. + construct(dynamic_lencode, dynamic_lengths, 19) + dynamic_index = 0 + + become(decode, call_decode(dynamic_lencode), on_got_dynamic_symbol_iter) + return + } + become(bits, call_bits(3), on_got_lengths_part) + } + + function on_got_dynamic_symbol_iter() { + dynamic_symbol = last + + if(dynamic_symbol < 16) { + dynamic_lengths[dynamic_index++] = dynamic_symbol + do_check() + return + } + + dynamic_len = 0 + if(dynamic_symbol === 16) { + become(bits, call_bits(2), on_got_dynamic_symbol_16) + return + } + + if(dynamic_symbol === 17) { + become(bits, call_bits(3), on_got_dynamic_symbol_17) + return + } + + become(bits, call_bits(7), on_got_dynamic_symbol) + } + + function on_got_dynamic_symbol_16() { + dynamic_len = dynamic_lengths[dynamic_index - 1] + on_got_dynamic_symbol_17() + } + + function on_got_dynamic_symbol_17() { + dynamic_symbol = 3 + last + do_dynamic_end_loop() + } + + function on_got_dynamic_symbol() { + dynamic_symbol = 11 + last + do_dynamic_end_loop() + } + + function do_dynamic_end_loop() { + if(dynamic_index + dynamic_symbol > dynamic_nlen + dynamic_ndist) { + emit(new Error('too many lengths')) + return + } + + while(dynamic_symbol--) { + dynamic_lengths[dynamic_index++] = dynamic_len + } + + do_check() + } + + function do_check() { + if(dynamic_index >= dynamic_nlen + dynamic_ndist) { + end_read_dynamic() + return + } + become(decode, call_decode(dynamic_lencode), on_got_dynamic_symbol_iter) + } + + function end_read_dynamic() { + // okay, we can finally start reading data out of the stream. + construct(dynamic_lencode, dynamic_lengths, dynamic_nlen) + construct(dynamic_distcode, dynamic_lengths.slice(dynamic_nlen), dynamic_ndist) + become(start_codes, call_codes( + dynamic_distcode + , dynamic_lencode + ), done_with_codes) + } + + function start_codes() { + become(decode, call_decode(codes_lencode), on_got_codes_symbol) + } + + function on_got_codes_symbol() { + var symbol = codes_symbol = last + if(symbol < 0) { + emit(new Error('invalid symbol')) + return + } + + if(symbol < 256) { + output_one(symbol) + become(decode, call_decode(codes_lencode), on_got_codes_symbol) + return + } + + if(symbol > 256) { + symbol = codes_symbol -= 257 + if(symbol >= 29) { + emit(new Error('invalid fixed code')) + return + } + + become(bits, call_bits(lext[symbol]), on_got_codes_len) + return + } + + if(symbol === 256) { + unbecome() + return + } + } + + + + + + + function on_got_codes_len() { + codes_len = lens[codes_symbol] + last + become(decode, call_decode(codes_distcode), on_got_codes_dist_symbol) + } + + + function on_got_codes_dist_symbol() { + codes_symbol = last + if(codes_symbol < 0) { + emit(new Error('invalid distance symbol')) + return + } + + become(bits, call_bits(dext[codes_symbol]), on_got_codes_dist_dist) + } + + function on_got_codes_dist_dist() { + var dist = dists[codes_symbol] + last + + // Once we have a "distance" and a "length", we start to output bytes. + // We reach "dist" back from our current output position to get the byte + // we should repeat and output it (thus moving the output window cursor forward). + // Two notes: + // + // 1. Theoretically we could overlap our output and input. + // 2. `X % (2^N) == X & (2^N - 1)` with the distinction that + // the result of the bitwise AND won't be negative for the + // range of values we're feeding it. Spare a modulo, spoil the child. + while(codes_len--) { + output_one(output[(output_idx - dist) & WINDOW_MINUS_ONE]) + } + + become(decode, call_decode(codes_lencode), on_got_codes_symbol) + } + + function done_with_codes() { + if(is_final) { + become(bytes, call_bytes(4), on_got_adler) + return + } + become(bits, call_bits(1), on_got_is_final) + } + + + + + function on_got_adler() { + var check_s1 = last[3] | (last[2] << 8) + , check_s2 = last[1] | (last[0] << 8) + + if(check_s2 !== adler_s2 || check_s1 !== adler_s1) { + emit(new Error( + 'bad adler checksum: '+[check_s2, adler_s2, check_s1, adler_s1] + )) + return + } + + ended = true + + output_one_recycle() + + if(on_unused) { + on_unused( + [binary.subarray(buffer[0], buffer_offset)].concat(buffer.slice(1)) + , bytes_read + ) + } + + output_idx = 0 + ended = true + emit() + } + + function decode() { + _decode() + } + + function _decode() { + if(decode_len > MAXBITS) { + emit(new Error('ran out of codes')) + return + } + + become(bits, call_bits(1), got_decode_bit) + } + + function got_decode_bit() { + decode_code = (decode_code | last) >>> 0 + decode_count = decode_huffman.count[decode_len] + if(decode_code < decode_first + decode_count) { + unbecome(decode_huffman.symbol[decode_index + (decode_code - decode_first)]) + return + } + decode_index += decode_count + decode_first += decode_count + decode_first <<= 1 + decode_code = (decode_code << 1) >>> 0 + ++decode_len + _decode() + } + + + function become(fn, s, then) { + if(typeof then !== 'function') { + throw new Error + } + states.unshift({ + current: fn + , next: then + }) + } + + function unbecome(result) { + if(states.length > 1) { + states[1].current = states[0].next + } + states.shift() + if(!states.length) { + ended = true + + output_one_recycle() + if(on_unused) { + on_unused( + [binary.subarray(buffer[0], buffer_offset)].concat(buffer.slice(1)) + , bytes_read + ) + } + output_idx = 0 + ended = true + emit() + // return + } + else { + last = result + } + } + + function bits() { + var byt + , idx + + idx = 0 + bits_value = bitbuf + while(bitcnt < bits_need) { + // we do this to preserve `bits_value` when + // "need_input" is tripped. + // + // fun fact: if we moved that into the `if` statement + // below, it would trigger a deoptimization of this (very + // hot) function. JITs! + bitbuf = bits_value + byt = take() + if(need_input) { + break + } + ++idx + bits_value = (bits_value | (byt << bitcnt)) >>> 0 + bitcnt += 8 + } + + if(!need_input) { + bitbuf = bits_value >>> bits_need + bitcnt -= bits_need + unbecome((bits_value & ((1 << bits_need) - 1)) >>> 0) + } + } + + + + function bytes() { + var byte_accum = bytes_value + , value + + while(bytes_need--) { + value = take() + + + if(need_input) { + bitbuf = bitcnt = 0 + bytes_need += 1 + break + } + byte_accum[byte_accum.length] = value + } + if(!need_input) { + bitcnt = bitbuf = 0 + unbecome(byte_accum) + } + } + + + + function take() { + if(!buffer.length) { + need_input = true + return + } + + if(buffer_offset === buffer[0].length) { + buffer.shift() + buffer_offset = 0 + return take() + } + + ++bytes_read + + return bitbuf = takebyte() + } + + function takebyte() { + return buffer[0][buffer_offset++] + } + + + + function output_one(val) { + adler_s1 = (adler_s1 + val) % 65521 + adler_s2 = (adler_s2 + adler_s1) % 65521 + output[output_idx++] = val + output_idx &= WINDOW_MINUS_ONE + output_one_pool(val) + } + + function output_one_pool(val) { + if(output_one_offs === OUTPUT_ONE_LENGTH) { + output_one_recycle() + } + + output_one_buf[output_one_offs++] = val + } + + function output_one_recycle() { + if(output_one_offs > 0) { + if(output_one_buf) { + emit(null, binary.subarray(output_one_buf, 0, output_one_offs)) + } else { + } + output_one_buf = binary.create(OUTPUT_ONE_LENGTH) + output_one_offs = 0 + } + } + + function output_many(vals) { + var len + , byt + , olen + + output_one_recycle() + for(var i = 0, len = vals.length; i < len; ++i) { + byt = vals[i] + adler_s1 = (adler_s1 + byt) % 65521 + adler_s2 = (adler_s2 + adler_s1) % 65521 + output[output_idx++] = byt + output_idx &= WINDOW_MINUS_ONE + } + + emit(null, binary.from(vals)) + } + } + + function build_fixed() { + var lencnt = [] + , lensym = [] + , distcnt = [] + , distsym = [] + + var lencode = { + count: lencnt + , symbol: lensym + } + + var distcode = { + count: distcnt + , symbol: distsym + } + + var lengths = [] + , symbol + + for(symbol = 0; symbol < 144; ++symbol) { + lengths[symbol] = 8 + } + for(; symbol < 256; ++symbol) { + lengths[symbol] = 9 + } + for(; symbol < 280; ++symbol) { + lengths[symbol] = 7 + } + for(; symbol < FIXLCODES; ++symbol) { + lengths[symbol] = 8 + } + construct(lencode, lengths, FIXLCODES) + + for(symbol = 0; symbol < MAXDCODES; ++symbol) { + lengths[symbol] = 5 + } + construct(distcode, lengths, MAXDCODES) + return {lencode: lencode, distcode: distcode} + } + + function construct(huffman, lengths, num) { + var symbol + , left + , offs + , len + + offs = [] + + for(len = 0; len <= MAXBITS; ++len) { + huffman.count[len] = 0 + } + + for(symbol = 0; symbol < num; ++symbol) { + huffman.count[lengths[symbol]] += 1 + } + + if(huffman.count[0] === num) { + return + } + + left = 1 + for(len = 1; len <= MAXBITS; ++len) { + left <<= 1 + left -= huffman.count[len] + if(left < 0) { + return left + } + } + + offs[1] = 0 + for(len = 1; len < MAXBITS; ++len) { + offs[len + 1] = offs[len] + huffman.count[len] + } + + for(symbol = 0; symbol < num; ++symbol) { + if(lengths[symbol] !== 0) { + huffman.symbol[offs[lengths[symbol]]++] = symbol + } + } + + return left + } + + return inflate; + +}; \ No newline at end of file diff --git a/lib/pkt-line.js b/lib/pkt-line.js new file mode 100644 index 0000000..8d4a279 --- /dev/null +++ b/lib/pkt-line.js @@ -0,0 +1,153 @@ +var binary = require('./binary.js'); +var PACK = binary.build("PACK"); + +module.exports = { + deframer: deframer, + framer: framer, + frame: frame +}; + +function deframer(emit) { + var offset, length, data, string, type; + var state = reset(); + var working = false; + + return function (chunk) { + if (working) throw new Error("parser is not re-entrant"); + working = true; + // Once in raw mode, pass all data through. + if (!state) return emit("pack", chunk); + + for (var i = 0, l = chunk.length; i < l; i++) { + state = state(chunk[i]); + if (state) continue; + emit("pack", binary.subarary(chunk, i)); + break; + } + working = false; + }; + + function reset() { + offset = 4; + length = 0; + data = null; + string = ""; + type = undefined; + return $len; + } + + function $len(byte) { + var val = fromHexChar(byte); + // If the byte isn't a hex char, it's bad input or it's raw PACK data. + if (val === -1) { + if (byte === PACK[0]) { + offset = 1; + return $maybeRaw; + } + throw new SyntaxError("Not a hex char: " + String.fromCharCode(byte)); + } + length |= val << ((--offset) * 4); + if (offset) return $len; + if (length === 0) { + offset = 4; + emit("line", null); + return $len; + } + if (length === 4) { + offset = 4; + emit("line", ""); + return $len; + } + if (length < 4) { + throw new SyntaxError("Invalid length: " + length); + } + length -= 4; + return $firstByte; + } + + function $firstByte(byte) { + if (byte === 1) { + // PACK data in side-band + length--; + type = "pack"; + data = new Uint8Array(length); + return $binary; + } + if (byte === 2) { + length--; + type = "progress"; + string = ""; + return $string; + } + if (byte === 3) { + length--; + type = "error"; + string = ""; + return $string; + } + type = "line"; + string = String.fromCharCode(byte); + return $string; + } + + function $binary(byte) { + data[offset] = byte; + if (++offset < length) return $binary; + emit(type, data); + return reset(); + } + + function $string(byte) { + string += String.fromCharCode(byte); + if (++offset < length) return $string; + emit(type, string); + return reset(); + } + + function $maybeRaw(byte) { + if (offset === 4) return; + if (byte === PACK[offset++]) return $maybeRaw; + throw new SyntaxError("Invalid data in raw pack header"); + } +} + +function framer(emit) { + return function (type, value) { + emit(frame(type, value)); + }; +} + +// Strings in line, progress, and error messages are assumed to be binary +// encoded. If you want to send unicode data, please utf8 encode first. +function frame(type, value) { + if (value === undefined && binary.is(type)) return type; + if (type === "line") { + if (value === null) return binary.build("0000"); + return binary.build(hexLen(value.length + 4) + value); + } + if (type === "pack") { + return binary.build(hexLen(value.length + 5) + "\u0001", value); + } + if (type == "progress") { + return binary.build(hexLen(value.length + 5) + "\u0002" + value); + } + if (type == "error") { + return binary.build(hexLen(value.length + 5) + "\u0003" + value); + } + throw new Error("Unknown type: " + type); +} + +// Given the code for a hex character, return it's value. +function fromHexChar(val) { + return val >= 0x30 && val < 0x40 ? val - 0x30 : + val > 0x60 && val <= 0x66 ? val - 0x57 : -1; +} + +// Given a number, return a 4-digit hex number +function hexLen(length) { + if (length > 0xffff) throw new Error("Length too large"); + return (length >> 12 & 0xf).toString(16) + + (length >> 8 & 0xf).toString(16) + + (length >> 4 & 0xf).toString(16) + + (length & 0xf).toString(16); +} diff --git a/lib/string-encodings.js b/lib/string-encodings.js new file mode 100644 index 0000000..1b817a7 --- /dev/null +++ b/lib/string-encodings.js @@ -0,0 +1,44 @@ +/*global escape, unescape*/ +// NOTE: This JS file *must* be served with a utf8 mime-type. + +module.exports = { + utf8: { + encode: utf8Encode, + decode: utf8Decode, + }, + hex: { + encode: hexEncode, + decode: hexDecode, + }, + // base64: { + // encode: base64Encode, + // decode: base64Decode, + // }, +}; + +function utf8Encode(string) { + return unescape(encodeURIComponent(string)); +} + +function utf8Decode(string) { + return decodeURIComponent(escape(string)); +} + +function hexEncode(string) { + var out = ""; + for (var i = 0, l = string.length; i < l; i++) { + var code = string.charCodeAt(i) & 0xff; + if (code < 0x10) out += "0" + code.toString(16); + out += code.toString(16); + } + return out; +} + +function hexDecode(string) { + var out = ""; + for (var i = 0, l = string.length; i < l; i += 2) { + var code = parseInt(string.substr(i, 2), 16); + out += String.fromCharCode(code & 0xff); + } + return out; +}