Mercurial > touhou

changeset 252:b5c7369abd7c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Improve data reading perfs
author Thibaut Girka <thib@sitedethib.com>
date Sun, 22 Jan 2012 15:54:51 +0100
parents 4b549894ef6b
children ea4832f843aa
files pytouhou/formats/pbg3.py pytouhou/utils/bitstream.pyx pytouhou/utils/lzss.py pytouhou/utils/lzss.pyx
diffstat 4 files changed, 65 insertions(+), 44 deletions(-) [+]
line wrap: on
line diff
--- a/pytouhou/formats/pbg3.py	Sun Jan 22 14:24:38 2012 +0100
+++ b/pytouhou/formats/pbg3.py	Sun Jan 22 15:54:51 2012 +0100
@@ -79,7 +79,7 @@
     """
 
     def __init__(self, entries=None, bitstream=None):
-        self.entries = entries or []
+        self.entries = entries or {}
         self.bitstream = bitstream #TODO
 
 
--- a/pytouhou/utils/bitstream.pyx	Sun Jan 22 14:24:38 2012 +0100
+++ b/pytouhou/utils/bitstream.pyx	Sun Jan 22 15:54:51 2012 +0100
@@ -15,7 +15,8 @@
 cdef class BitStream:
     cdef public io
     cdef public int bits
-    cdef public int byte
+    cdef public unsigned char byte
+
 
     def __init__(BitStream self, io):
         self.io = io
@@ -53,9 +54,10 @@
         return (self.byte >> self.bits) & 0x01
 
 
-    def read(BitStream self, nb_bits):
-        cdef unsigned int value
-        value = 0
+    cpdef unsigned int read(BitStream self, int nb_bits):
+        cdef unsigned int value = 0
+        cdef int i
+
         for i in range(nb_bits - 1, -1, -1):
             value |= self.read_bit() << i
         return value
--- a/pytouhou/utils/lzss.py	Sun Jan 22 14:24:38 2012 +0100
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,39 +0,0 @@
-# -*- encoding: utf-8 -*-
-##
-## Copyright (C) 2011 Thibaut Girka <thib@sitedethib.com>
-##
-## This program is free software; you can redistribute it and/or modify
-## it under the terms of the GNU General Public License as published
-## by the Free Software Foundation; version 3 only.
-##
-## This program is distributed in the hope that it will be useful,
-## but WITHOUT ANY WARRANTY; without even the implied warranty of
-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-## GNU General Public License for more details.
-##
-
-def decompress(bitstream, size, dictionary_size=0x2000,
-               offset_size=13, length_size=4, minimum_match_length=3):
-    out_data = []
-    dictionary = [0] * dictionary_size
-    dictionary_head = 1
-    while len(out_data) < size:
-        flag = bitstream.read_bit()
-        if flag:
-            # The `flag` bit is set, indicating the upcoming chunk of data is a literal
-            # Add it to the uncompressed file, and store it in the dictionary
-            byte = bitstream.read(8)
-            dictionary[dictionary_head] = byte
-            dictionary_head = (dictionary_head + 1) % dictionary_size
-            out_data.append(byte)
-        else:
-            # The `flag` bit is not set, the upcoming chunk is a (offset, length) tuple
-            offset = bitstream.read(offset_size)
-            length = bitstream.read(length_size) + minimum_match_length
-            if (offset, length) == (0, 0):
-                break
-            for i in range(offset, offset + length):
-                out_data.append(dictionary[i % dictionary_size])
-                dictionary[dictionary_head] = dictionary[i % dictionary_size]
-                dictionary_head = (dictionary_head + 1) % dictionary_size
-    return b''.join(chr(byte) for byte in out_data)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pytouhou/utils/lzss.pyx	Sun Jan 22 15:54:51 2012 +0100
@@ -0,0 +1,58 @@
+# -*- encoding: utf-8 -*-
+##
+## Copyright (C) 2011 Thibaut Girka <thib@sitedethib.com>
+##
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published
+## by the Free Software Foundation; version 3 only.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+##
+
+from libc.stdlib cimport calloc, malloc, free
+
+
+cpdef bytes decompress(object bitstream,
+                       Py_ssize_t size,
+                       unsigned int dictionary_size=0x2000,
+                       unsigned int offset_size=13,
+                       unsigned int length_size=4,
+                       unsigned int minimum_match_length=3):
+    cdef unsigned int i, ptr, dictionary_head, offset, length
+    cdef unsigned char flag, byte, *out_data, *dictionary
+    cdef bytes _out_data
+
+    out_data = <unsigned char*> malloc(size)
+    dictionary = <unsigned char*> calloc(dictionary_size, 1)
+    dictionary_head, ptr = 1, 0
+
+    while ptr < size:
+        flag = bitstream.read_bit()
+        if flag:
+            # The `flag` bit is set, indicating the upcoming chunk of data is a literal
+            # Add it to the uncompressed file, and store it in the dictionary
+            byte = bitstream.read(8)
+            dictionary[dictionary_head] = byte
+            dictionary_head = (dictionary_head + 1) % dictionary_size
+            out_data[ptr] = byte
+            ptr += 1
+        else:
+            # The `flag` bit is not set, the upcoming chunk is a (offset, length) tuple
+            offset = bitstream.read(offset_size)
+            length = bitstream.read(length_size) + minimum_match_length
+            if offset == 0 and length == 0:
+                break
+            for i in range(offset, offset + length):
+                out_data[ptr % size] = dictionary[i % dictionary_size]
+                ptr += 1
+                dictionary[dictionary_head] = dictionary[i % dictionary_size]
+                dictionary_head = (dictionary_head + 1) % dictionary_size
+
+    _out_data = out_data[:size]
+    free(out_data)
+    free(dictionary)
+    return _out_data
+