From 284ea946b599bdb2f08f7481fa2ce7a6767e1941 Mon Sep 17 00:00:00 2001 From: Antoine Le Calvez Date: Fri, 18 Sep 2020 16:32:46 +0200 Subject: [PATCH 01/26] Only run build on 3.6 --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d7b170f..34f2a4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,9 +4,6 @@ language: - python python: - - "3.3" - - "3.4" - - "3.5" - "3.6" before_install: From ee5b9e2e28a0c0848d8d46b1c80d90549933fb71 Mon Sep 17 00:00:00 2001 From: Antoine Le Calvez Date: Tue, 22 Sep 2020 16:22:34 +0200 Subject: [PATCH 02/26] Fix Readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c57701b..3f79a48 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ from blockchain_parser.blockchain import Blockchain # To get the blocks ordered by height, you need to provide the path of the # `index` directory (LevelDB index) being maintained by bitcoind. It contains # .ldb files and is present inside the `blocks` directory. +blockchain = Blockchain(os.path.expanduser('~/.bitcoin/blocks')) for block in blockchain.get_ordered_blocks(os.path.expanduser('~/.bitcoin/blocks/index'), end=1000): print("height=%d block=%s" % (block.height, block.hash)) ``` From dea28f2f4f487b8ab7cac4f233bda732fde93a25 Mon Sep 17 00:00:00 2001 From: Antoine Le Calvez Date: Wed, 23 Sep 2020 10:19:54 +0200 Subject: [PATCH 03/26] Move test data to files --- .../tests/{ => data}/bech32_p2wpkh.txt | 0 .../tests/{ => data}/bech32_p2wsh.txt | 0 .../tests/{ => data}/bip69_false.txt | 0 .../tests/{ => data}/bip69_true.txt | 0 .../tests/data/genesis_block.txt | 1 + blockchain_parser/tests/data/invalid_tx.txt | 1 + blockchain_parser/tests/data/large_tx.txt | 1 + .../tests/{ => data}/scripts_invalid.txt | 0 blockchain_parser/tests/{ => data}/segwit.txt | 0 .../tests/{ => data}/size_non_segwit.txt | 0 .../tests/{ => data}/size_segwit.txt | 0 blockchain_parser/tests/test_block.py | 15 +- blockchain_parser/tests/test_transaction.py | 207 ++---------------- blockchain_parser/tests/utils.py | 9 + 14 files changed, 32 insertions(+), 202 deletions(-) rename blockchain_parser/tests/{ => data}/bech32_p2wpkh.txt (100%) rename blockchain_parser/tests/{ => data}/bech32_p2wsh.txt (100%) rename blockchain_parser/tests/{ => data}/bip69_false.txt (100%) rename blockchain_parser/tests/{ => data}/bip69_true.txt (100%) create mode 100644 blockchain_parser/tests/data/genesis_block.txt create mode 100644 blockchain_parser/tests/data/invalid_tx.txt create mode 100644 blockchain_parser/tests/data/large_tx.txt rename blockchain_parser/tests/{ => data}/scripts_invalid.txt (100%) rename blockchain_parser/tests/{ => data}/segwit.txt (100%) rename blockchain_parser/tests/{ => data}/size_non_segwit.txt (100%) rename blockchain_parser/tests/{ => data}/size_segwit.txt (100%) create mode 100644 blockchain_parser/tests/utils.py diff --git a/blockchain_parser/tests/bech32_p2wpkh.txt b/blockchain_parser/tests/data/bech32_p2wpkh.txt similarity index 100% rename from blockchain_parser/tests/bech32_p2wpkh.txt rename to blockchain_parser/tests/data/bech32_p2wpkh.txt diff --git a/blockchain_parser/tests/bech32_p2wsh.txt b/blockchain_parser/tests/data/bech32_p2wsh.txt similarity index 100% rename from blockchain_parser/tests/bech32_p2wsh.txt rename to blockchain_parser/tests/data/bech32_p2wsh.txt diff --git a/blockchain_parser/tests/bip69_false.txt b/blockchain_parser/tests/data/bip69_false.txt similarity index 100% rename from blockchain_parser/tests/bip69_false.txt rename to blockchain_parser/tests/data/bip69_false.txt diff --git a/blockchain_parser/tests/bip69_true.txt b/blockchain_parser/tests/data/bip69_true.txt similarity index 100% rename from blockchain_parser/tests/bip69_true.txt rename to blockchain_parser/tests/data/bip69_true.txt diff --git a/blockchain_parser/tests/data/genesis_block.txt b/blockchain_parser/tests/data/genesis_block.txt new file mode 100644 index 0000000..d15b142 --- /dev/null +++ b/blockchain_parser/tests/data/genesis_block.txt @@ -0,0 +1 @@ +0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000 \ No newline at end of file diff --git a/blockchain_parser/tests/data/invalid_tx.txt b/blockchain_parser/tests/data/invalid_tx.txt new file mode 100644 index 0000000..c618b59 --- /dev/null +++ b/blockchain_parser/tests/data/invalid_tx.txt @@ -0,0 +1 @@ +010000000b04a4abee8360f7a660bcf1c298496d927b6ad1f25dd12485e6a572e275cbd43f000000008b483045022100c6d59290df934bac08bc4124b06154ee21394d37ec958d145129a885dbd63eab0220036251120e4ceacc8e9ff13585049945c7dbdb85921a6e077a80c7e410931f4a014104f3ce8948c592690cb36a7e6e31534927569f41e45c77fbdb95fdb0e235717a295b2a281c04619139c453dd9139e951086325b4021bd28da6fc3da86943cd04cfffffffff21fd91e6dd64838eadce9bc0d687af571779c3af4b53bada845485466decf005000000008b48304502205d309b632a816f1f2479b8ca284301fce3fc8f979306722bba4cf28c8fa7ed76022100a3313ee59bbf0e61e29302b2ea597a993a73e93be841f2f7dd3f6e9623572793014104ca3b25750307a483c0cff8efb1413c98b4e18765485d94eac147f21f507169997a72aeb50f0b01ed6b468c9554143125f63c0b2b229a597e695718b86d711607ffffffff4dd6c46e3e68fc05243b6f894e2f3e4d2782d1c8a3d17b9a2c5e78912bbaf433010000008b483045022100f3f1d6c0e76a00e89a5278fa14eae8de93454174199f72c98aef54dabb2074d10220172dc0d6958efa8dc18c621f2d56fd0484ecb04e3d65e2284268916c1c324ade014104daeab6af397baf06fcc91b6c04a28ffb2cd80312e954a2a55f7c864094608032ef657e8296dfc2f86ecf28fe32925c0f648d3ef16b96934a34b29dd871ff057affffffff52aabc7fdee0588d63fff3fc00e20b00a87cffa6a37aad7168741b1f0e77bdff000000008b4830450220274a5a9577f0786fb17d373b6d1995a6cff752320bf4e3d5ee41b15fdd86ef2a022100fd1d0c0f43d05e479430c3c9bd5481f5be929a7f2b82d14292ef192653c3b94c014104988828ea7144ff00def9988e88aad98b1f22d01000fc4b52d1003c9b32de0534cff71c0a045098bb28f699bcc9db13533c0998eb2133085d37cac65b5bd74c1fffffffff5aab5581c575a56a803ef5d8698dc34014261d196ada1da3300731380d7e672d000000008b48304502201615a116a2e057f3f8f2a63340d0387b820a6f9d1067758f4f6cdec76c04951a022100a1ada9e03a0d9603743d41347ffd992642544d6dfdc51121b7bf622e12cb1547014104830b3b139b0a8707f348840b0fa358164869fbc0c423a2785cb0a18ff73517f8878cefd643e86e682eb93d07216f76993c9e980592de07c507c3a2659efff6aaffffffff74d7cd370a334aabee3b8328b67a63f1768583405e73ebd291dd86189983a61c000000008a4730440220432ff0f90a3f5372a8dc4b3d2e7165e110efa16b40784ea4d5ab866b736712ed02201a759b630d562f4dbc4a7d2fca406ad54e235564346bc360dec26d6e72bb60fa014104ca0bb72303672885ad953279309694c71574d50d2c6e1f5f1e7ab1b0bf3df67e5439b3f6e2501dda539812e414991ebaa547631374ce2bda9dd7211099886827ffffffff8941eb7fe688ff48357981c4756906e331c07f332429be5b2cfc7a945644f80f000000008a47304402204e074678b525927a8f459d1dc93ad738ce02c0ea77c50a749647032eda662eee02203c1a98e9b116dc2fe9e7325a5696ad8e3ad26d2d10503e6e3209f4f5f40f6e34014104f9baa86b46b04f2a2dc4270903bcca9444026b7f184d879f089462e332f260f6de8e05f08285a4a3c716220d9ec0f7e9fde354c779f32f00dd4d4cd8d4a1ac28ffffffff948be2e03f9537a35319e8cee2b7b9ed2769ca59734e5d4b42951ef7ba3bb4a8000000008b483045022100fcf1deaaa047e8f1b8b8ac783760ea5a0af15adbb9eed1c1748942a57510e27402204c278b12c4991480dc714af66395f5d8f42e411423bf27b024f62267c45a0d37014104cfd128c3f322b59c21d452d5c4c7b4d1a3237a2d31c6f4033b32db78c19cb342ffc151acc272a02a13ba985a523b655689d8118f502b8a7fb8df5fc7bd8458ceffffffffb2cae04cf238b4d47c614da5ce9948f089a02f1ff3e22cda6b9f9196bda87215010000008b483045022017a74373ef19756e275f2d987925a8bd22cfaf1c93eede7db63102dc73b7cfd0022100fd2f040d7d3ccd1a2188923f8c7560253aa2a910d8e9cd06a8a05fc69c12f6ce014104daeab6af397baf06fcc91b6c04a28ffb2cd80312e954a2a55f7c864094608032ef657e8296dfc2f86ecf28fe32925c0f648d3ef16b96934a34b29dd871ff057affffffffbe75c9f1faf79fb885018e3353ada4a7fb5c989bc26f031c5aee69d1eabad957000000008a4730440220360d35b0a98dd086d242f8976aa704b808541ad78ef5cba45e4eb73c1efbaaa7022025a3d981b9e0836a2493698b802bf7f0eae72de0252c7eac059ad14b1b27001b014104d969f0d456196f4c16945bc71fc460615002d1554fc8d774288aa8fc5c9eca1622a3dcb1e6a8314c94844e516c0543316b2e75d465c8fc2f0d0d20334e53f7dbffffffffbcbad95cac8bbe08538f7b12c240e96866358dfa819de6e940bb60ce7fae5c3d000000008b483045022100d916990888d0d26246b8d6cb944b1e8ebd6a4cdb2965ff7e4430883eef02c4e102205693b96aa1d6e9335f3cb2986b7f3a03357959e989570d03a56ccdb18a8d7e6d0141040840a410ed7afdc3e3fe3923ae388413bbd8a9089062403e0adce62212bccfe1f80400530121cfdba5b52f3e16397f1959b14a09089778afd5ff41f8a856980fffffffff02005039278c0400001976a914f1c87a5e8ff7d14e74b858089bf771c94b1b6db488ac00203d88792d00001976a914dcdf2f9892bfa1cb086530354eab3ba078a2f0 \ No newline at end of file diff --git a/blockchain_parser/tests/data/large_tx.txt b/blockchain_parser/tests/data/large_tx.txt new file mode 100644 index 0000000..c06deab --- /dev/null +++ b/blockchain_parser/tests/data/large_tx.txt @@ -0,0 +1 @@ +010000000b04a4abee8360f7a660bcf1c298496d927b6ad1f25dd12485e6a572e275cbd43f000000008b483045022100c6d59290df934bac08bc4124b06154ee21394d37ec958d145129a885dbd63eab0220036251120e4ceacc8e9ff13585049945c7dbdb85921a6e077a80c7e410931f4a014104f3ce8948c592690cb36a7e6e31534927569f41e45c77fbdb95fdb0e235717a295b2a281c04619139c453dd9139e951086325b4021bd28da6fc3da86943cd04cfffffffff21fd91e6dd64838eadce9bc0d687af571779c3af4b53bada845485466decf005000000008b48304502205d309b632a816f1f2479b8ca284301fce3fc8f979306722bba4cf28c8fa7ed76022100a3313ee59bbf0e61e29302b2ea597a993a73e93be841f2f7dd3f6e9623572793014104ca3b25750307a483c0cff8efb1413c98b4e18765485d94eac147f21f507169997a72aeb50f0b01ed6b468c9554143125f63c0b2b229a597e695718b86d711607ffffffff4dd6c46e3e68fc05243b6f894e2f3e4d2782d1c8a3d17b9a2c5e78912bbaf433010000008b483045022100f3f1d6c0e76a00e89a5278fa14eae8de93454174199f72c98aef54dabb2074d10220172dc0d6958efa8dc18c621f2d56fd0484ecb04e3d65e2284268916c1c324ade014104daeab6af397baf06fcc91b6c04a28ffb2cd80312e954a2a55f7c864094608032ef657e8296dfc2f86ecf28fe32925c0f648d3ef16b96934a34b29dd871ff057affffffff52aabc7fdee0588d63fff3fc00e20b00a87cffa6a37aad7168741b1f0e77bdff000000008b4830450220274a5a9577f0786fb17d373b6d1995a6cff752320bf4e3d5ee41b15fdd86ef2a022100fd1d0c0f43d05e479430c3c9bd5481f5be929a7f2b82d14292ef192653c3b94c014104988828ea7144ff00def9988e88aad98b1f22d01000fc4b52d1003c9b32de0534cff71c0a045098bb28f699bcc9db13533c0998eb2133085d37cac65b5bd74c1fffffffff5aab5581c575a56a803ef5d8698dc34014261d196ada1da3300731380d7e672d000000008b48304502201615a116a2e057f3f8f2a63340d0387b820a6f9d1067758f4f6cdec76c04951a022100a1ada9e03a0d9603743d41347ffd992642544d6dfdc51121b7bf622e12cb1547014104830b3b139b0a8707f348840b0fa358164869fbc0c423a2785cb0a18ff73517f8878cefd643e86e682eb93d07216f76993c9e980592de07c507c3a2659efff6aaffffffff74d7cd370a334aabee3b8328b67a63f1768583405e73ebd291dd86189983a61c000000008a4730440220432ff0f90a3f5372a8dc4b3d2e7165e110efa16b40784ea4d5ab866b736712ed02201a759b630d562f4dbc4a7d2fca406ad54e235564346bc360dec26d6e72bb60fa014104ca0bb72303672885ad953279309694c71574d50d2c6e1f5f1e7ab1b0bf3df67e5439b3f6e2501dda539812e414991ebaa547631374ce2bda9dd7211099886827ffffffff8941eb7fe688ff48357981c4756906e331c07f332429be5b2cfc7a945644f80f000000008a47304402204e074678b525927a8f459d1dc93ad738ce02c0ea77c50a749647032eda662eee02203c1a98e9b116dc2fe9e7325a5696ad8e3ad26d2d10503e6e3209f4f5f40f6e34014104f9baa86b46b04f2a2dc4270903bcca9444026b7f184d879f089462e332f260f6de8e05f08285a4a3c716220d9ec0f7e9fde354c779f32f00dd4d4cd8d4a1ac28ffffffff948be2e03f9537a35319e8cee2b7b9ed2769ca59734e5d4b42951ef7ba3bb4a8000000008b483045022100fcf1deaaa047e8f1b8b8ac783760ea5a0af15adbb9eed1c1748942a57510e27402204c278b12c4991480dc714af66395f5d8f42e411423bf27b024f62267c45a0d37014104cfd128c3f322b59c21d452d5c4c7b4d1a3237a2d31c6f4033b32db78c19cb342ffc151acc272a02a13ba985a523b655689d8118f502b8a7fb8df5fc7bd8458ceffffffffb2cae04cf238b4d47c614da5ce9948f089a02f1ff3e22cda6b9f9196bda87215010000008b483045022017a74373ef19756e275f2d987925a8bd22cfaf1c93eede7db63102dc73b7cfd0022100fd2f040d7d3ccd1a2188923f8c7560253aa2a910d8e9cd06a8a05fc69c12f6ce014104daeab6af397baf06fcc91b6c04a28ffb2cd80312e954a2a55f7c864094608032ef657e8296dfc2f86ecf28fe32925c0f648d3ef16b96934a34b29dd871ff057affffffffbe75c9f1faf79fb885018e3353ada4a7fb5c989bc26f031c5aee69d1eabad957000000008a4730440220360d35b0a98dd086d242f8976aa704b808541ad78ef5cba45e4eb73c1efbaaa7022025a3d981b9e0836a2493698b802bf7f0eae72de0252c7eac059ad14b1b27001b014104d969f0d456196f4c16945bc71fc460615002d1554fc8d774288aa8fc5c9eca1622a3dcb1e6a8314c94844e516c0543316b2e75d465c8fc2f0d0d20334e53f7dbffffffffbcbad95cac8bbe08538f7b12c240e96866358dfa819de6e940bb60ce7fae5c3d000000008b483045022100d916990888d0d26246b8d6cb944b1e8ebd6a4cdb2965ff7e4430883eef02c4e102205693b96aa1d6e9335f3cb2986b7f3a03357959e989570d03a56ccdb18a8d7e6d0141040840a410ed7afdc3e3fe3923ae388413bbd8a9089062403e0adce62212bccfe1f80400530121cfdba5b52f3e16397f1959b14a09089778afd5ff41f8a856980fffffffff02005039278c0400001976a914f1c87a5e8ff7d14e74b858089bf771c94b1b6db488ac00203d88792d00001976a914dcdf2f9892bfa1cb086530354eab3ba078a2f09088ac00000000 \ No newline at end of file diff --git a/blockchain_parser/tests/scripts_invalid.txt b/blockchain_parser/tests/data/scripts_invalid.txt similarity index 100% rename from blockchain_parser/tests/scripts_invalid.txt rename to blockchain_parser/tests/data/scripts_invalid.txt diff --git a/blockchain_parser/tests/segwit.txt b/blockchain_parser/tests/data/segwit.txt similarity index 100% rename from blockchain_parser/tests/segwit.txt rename to blockchain_parser/tests/data/segwit.txt diff --git a/blockchain_parser/tests/size_non_segwit.txt b/blockchain_parser/tests/data/size_non_segwit.txt similarity index 100% rename from blockchain_parser/tests/size_non_segwit.txt rename to blockchain_parser/tests/data/size_non_segwit.txt diff --git a/blockchain_parser/tests/size_segwit.txt b/blockchain_parser/tests/data/size_segwit.txt similarity index 100% rename from blockchain_parser/tests/size_segwit.txt rename to blockchain_parser/tests/data/size_segwit.txt diff --git a/blockchain_parser/tests/test_block.py b/blockchain_parser/tests/test_block.py index 3cdc540..4c66a43 100644 --- a/blockchain_parser/tests/test_block.py +++ b/blockchain_parser/tests/test_block.py @@ -10,26 +10,15 @@ # in the LICENSE file. import unittest -from binascii import a2b_hex from datetime import datetime +from .utils import read_test_data from blockchain_parser.block import Block class TestBlock(unittest.TestCase): def test_from_hex(self): - block_str = "0100000000000000000000000000000000000000000000000000000" \ - "000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81b" \ - "c3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c01010" \ - "0000001000000000000000000000000000000000000000000000000" \ - "0000000000000000ffffffff4d04ffff001d0104455468652054696" \ - "d65732030332f4a616e2f32303039204368616e63656c6c6f72206f" \ - "6e206272696e6b206f66207365636f6e64206261696c6f757420666" \ - "f722062616e6b73ffffffff0100f2052a01000000434104678afdb0" \ - "fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb" \ - "649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c70" \ - "2b6bf11d5fac00000000" - block_hex = a2b_hex(block_str) + block_hex = read_test_data("genesis_block.txt") block = Block.from_hex(block_hex) self.assertEqual(1, block.n_transactions) block_hash = "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1" \ diff --git a/blockchain_parser/tests/test_transaction.py b/blockchain_parser/tests/test_transaction.py index 72cf1f1..66e3cc3 100644 --- a/blockchain_parser/tests/test_transaction.py +++ b/blockchain_parser/tests/test_transaction.py @@ -14,8 +14,7 @@ from binascii import a2b_hex, b2a_hex from blockchain_parser.transaction import Transaction -dir_path = os.path.dirname(os.path.realpath(__file__)) - +from .utils import read_test_data class TestTransaction(unittest.TestCase): def test_rbf(self): @@ -37,44 +36,26 @@ def test_rbf(self): self.assertFalse(tx.uses_replace_by_fee()) def test_bip69(self): - noncompliant = "bip69_false.txt" - with open(os.path.join(dir_path, noncompliant)) as f: - data = a2b_hex(f.read().strip()) - - tx = Transaction(data) + non_compliant = read_test_data("bip69_false.txt") + tx = Transaction(non_compliant) self.assertFalse(tx.uses_bip69()) - compliant = "bip69_true.txt" - with open(os.path.join(dir_path, compliant)) as f: - data = a2b_hex(f.read().strip()) - - tx = Transaction(data) + compliant = read_test_data("bip69_true.txt") + tx = Transaction(compliant) self.assertTrue(tx.uses_bip69()) def test_bech32_p2wpkh(self): - example_tx = "bech32_p2wpkh.txt" - with open(os.path.join(dir_path, example_tx)) as f: - data = a2b_hex(f.read().strip()) - - tx = Transaction(data) + tx = Transaction(read_test_data("bech32_p2wpkh.txt")) self.assertEqual(["3BBqfnaPbgi5KWECWdFpvryUfw7QatWy37"], [a.address for a in tx.outputs[0].addresses]) self.assertEqual(["bc1q4z0874xmfxe3xeqknulgnqhukhfjwh5tvjrr2x"], [a.address for a in tx.outputs[1].addresses]) def test_bech32_p2wsh(self): - example_tx = "bech32_p2wsh.txt" - with open(os.path.join(dir_path, example_tx)) as f: - data = a2b_hex(f.read().strip()) - - tx = Transaction(data) + tx = Transaction(read_test_data("bech32_p2wsh.txt")) self.assertEqual(["3GMKKFPNUg13VktgihUD8QfXVQRBdoDNDf"], [a.address for a in tx.outputs[0].addresses]) self.assertEqual(["bc1qday7wsftyv4r6qkpn8907s8fy3kexkny0xwrd8d4wlk06zffzyuqpp629n"], [a.address for a in tx.outputs[1].addresses]) def test_segwit(self): - example_tx = "segwit.txt" - with open(os.path.join(dir_path, example_tx)) as f: - data = a2b_hex(f.read().strip()) - - tx = Transaction(data) + tx = Transaction(read_test_data("segwit.txt")) self.assertTrue(tx.is_segwit) id = "22116f1d76ab425ddc6d10d184331e70e080dd6275d7aa90237ceb648dc38224" self.assertTrue(tx.txid == id) @@ -107,97 +88,18 @@ def test_segwit(self): self.assertEqual(parsed_3, wit_3) def test_vsize(self): - file_segwit_tx = "size_segwit.txt" - file_non_segwit_tx = "size_non_segwit.txt" - - with open(os.path.join(dir_path, file_non_segwit_tx)) as f: - data = a2b_hex(f.read().strip()) + segwit_tx = Transaction(read_test_data("size_segwit.txt")) + non_segwit_tx = Transaction(read_test_data("size_non_segwit.txt")) - tx = Transaction(data) - self.assertEqual(tx.vsize, tx.size) - self.assertEqual(tx.vsize, 189) - - with open(os.path.join(dir_path, file_segwit_tx)) as f: - data = a2b_hex(f.read().strip()) + self.assertEqual(non_segwit_tx.vsize, non_segwit_tx.size) + self.assertEqual(non_segwit_tx.vsize, 189) - tx = Transaction(data) - self.assertNotEqual(tx.vsize, tx.size) - self.assertEqual(tx.vsize, 208) - self.assertEqual(tx.size, 373) + self.assertNotEqual(segwit_tx.vsize, segwit_tx.size) + self.assertEqual(segwit_tx.vsize, 208) + self.assertEqual(segwit_tx.size, 373) def test_large(self): - data = a2b_hex( - "010000000b04a4abee8360f7a660bcf1c298496d927b6ad1f25dd12485" - "e6a572e275cbd43f000000008b483045022100c6d59290df934bac08bc" - "4124b06154ee21394d37ec958d145129a885dbd63eab0220036251120e" - "4ceacc8e9ff13585049945c7dbdb85921a6e077a80c7e410931f4a0141" - "04f3ce8948c592690cb36a7e6e31534927569f41e45c77fbdb95fdb0e2" - "35717a295b2a281c04619139c453dd9139e951086325b4021bd28da6fc" - "3da86943cd04cfffffffff21fd91e6dd64838eadce9bc0d687af571779" - "c3af4b53bada845485466decf005000000008b48304502205d309b632a" - "816f1f2479b8ca284301fce3fc8f979306722bba4cf28c8fa7ed760221" - "00a3313ee59bbf0e61e29302b2ea597a993a73e93be841f2f7dd3f6e96" - "23572793014104ca3b25750307a483c0cff8efb1413c98b4e18765485d" - "94eac147f21f507169997a72aeb50f0b01ed6b468c9554143125f63c0b" - "2b229a597e695718b86d711607ffffffff4dd6c46e3e68fc05243b6f89" - "4e2f3e4d2782d1c8a3d17b9a2c5e78912bbaf433010000008b48304502" - "2100f3f1d6c0e76a00e89a5278fa14eae8de93454174199f72c98aef54" - "dabb2074d10220172dc0d6958efa8dc18c621f2d56fd0484ecb04e3d65" - "e2284268916c1c324ade014104daeab6af397baf06fcc91b6c04a28ffb" - "2cd80312e954a2a55f7c864094608032ef657e8296dfc2f86ecf28fe32" - "925c0f648d3ef16b96934a34b29dd871ff057affffffff52aabc7fdee0" - "588d63fff3fc00e20b00a87cffa6a37aad7168741b1f0e77bdff000000" - "008b4830450220274a5a9577f0786fb17d373b6d1995a6cff752320bf4" - "e3d5ee41b15fdd86ef2a022100fd1d0c0f43d05e479430c3c9bd5481f5" - "be929a7f2b82d14292ef192653c3b94c014104988828ea7144ff00def9" - "988e88aad98b1f22d01000fc4b52d1003c9b32de0534cff71c0a045098" - "bb28f699bcc9db13533c0998eb2133085d37cac65b5bd74c1fffffffff" - "5aab5581c575a56a803ef5d8698dc34014261d196ada1da3300731380d" - "7e672d000000008b48304502201615a116a2e057f3f8f2a63340d0387b" - "820a6f9d1067758f4f6cdec76c04951a022100a1ada9e03a0d9603743d" - "41347ffd992642544d6dfdc51121b7bf622e12cb1547014104830b3b13" - "9b0a8707f348840b0fa358164869fbc0c423a2785cb0a18ff73517f887" - "8cefd643e86e682eb93d07216f76993c9e980592de07c507c3a2659eff" - "f6aaffffffff74d7cd370a334aabee3b8328b67a63f1768583405e73eb" - "d291dd86189983a61c000000008a4730440220432ff0f90a3f5372a8dc" - "4b3d2e7165e110efa16b40784ea4d5ab866b736712ed02201a759b630d" - "562f4dbc4a7d2fca406ad54e235564346bc360dec26d6e72bb60fa0141" - "04ca0bb72303672885ad953279309694c71574d50d2c6e1f5f1e7ab1b0" - "bf3df67e5439b3f6e2501dda539812e414991ebaa547631374ce2bda9d" - "d7211099886827ffffffff8941eb7fe688ff48357981c4756906e331c0" - "7f332429be5b2cfc7a945644f80f000000008a47304402204e074678b5" - "25927a8f459d1dc93ad738ce02c0ea77c50a749647032eda662eee0220" - "3c1a98e9b116dc2fe9e7325a5696ad8e3ad26d2d10503e6e3209f4f5f4" - "0f6e34014104f9baa86b46b04f2a2dc4270903bcca9444026b7f184d87" - "9f089462e332f260f6de8e05f08285a4a3c716220d9ec0f7e9fde354c7" - "79f32f00dd4d4cd8d4a1ac28ffffffff948be2e03f9537a35319e8cee2" - "b7b9ed2769ca59734e5d4b42951ef7ba3bb4a8000000008b4830450221" - "00fcf1deaaa047e8f1b8b8ac783760ea5a0af15adbb9eed1c1748942a5" - "7510e27402204c278b12c4991480dc714af66395f5d8f42e411423bf27" - "b024f62267c45a0d37014104cfd128c3f322b59c21d452d5c4c7b4d1a3" - "237a2d31c6f4033b32db78c19cb342ffc151acc272a02a13ba985a523b" - "655689d8118f502b8a7fb8df5fc7bd8458ceffffffffb2cae04cf238b4" - "d47c614da5ce9948f089a02f1ff3e22cda6b9f9196bda8721501000000" - "8b483045022017a74373ef19756e275f2d987925a8bd22cfaf1c93eede" - "7db63102dc73b7cfd0022100fd2f040d7d3ccd1a2188923f8c7560253a" - "a2a910d8e9cd06a8a05fc69c12f6ce014104daeab6af397baf06fcc91b" - "6c04a28ffb2cd80312e954a2a55f7c864094608032ef657e8296dfc2f8" - "6ecf28fe32925c0f648d3ef16b96934a34b29dd871ff057affffffffbe" - "75c9f1faf79fb885018e3353ada4a7fb5c989bc26f031c5aee69d1eaba" - "d957000000008a4730440220360d35b0a98dd086d242f8976aa704b808" - "541ad78ef5cba45e4eb73c1efbaaa7022025a3d981b9e0836a2493698b" - "802bf7f0eae72de0252c7eac059ad14b1b27001b014104d969f0d45619" - "6f4c16945bc71fc460615002d1554fc8d774288aa8fc5c9eca1622a3dc" - "b1e6a8314c94844e516c0543316b2e75d465c8fc2f0d0d20334e53f7db" - "ffffffffbcbad95cac8bbe08538f7b12c240e96866358dfa819de6e940" - "bb60ce7fae5c3d000000008b483045022100d916990888d0d26246b8d6" - "cb944b1e8ebd6a4cdb2965ff7e4430883eef02c4e102205693b96aa1d6" - "e9335f3cb2986b7f3a03357959e989570d03a56ccdb18a8d7e6d014104" - "0840a410ed7afdc3e3fe3923ae388413bbd8a9089062403e0adce62212" - "bccfe1f80400530121cfdba5b52f3e16397f1959b14a09089778afd5ff" - "41f8a856980fffffffff02005039278c0400001976a914f1c87a5e8ff7" - "d14e74b858089bf771c94b1b6db488ac00203d88792d00001976a914dc" - "df2f9892bfa1cb086530354eab3ba078a2f09088ac00000000") + data = read_test_data("large_tx.txt") tx = Transaction(data) self.assertTrue( @@ -206,85 +108,12 @@ def test_large(self): self.assertTrue(tx.size == len(tx.hex)) def test_incomplete(self): - data = a2b_hex( - "010000000b04a4abee8360f7a660bcf1c298496d927b6ad1f25dd12485" - "e6a572e275cbd43f000000008b483045022100c6d59290df934bac08bc" - "4124b06154ee21394d37ec958d145129a885dbd63eab0220036251120e" - "4ceacc8e9ff13585049945c7dbdb85921a6e077a80c7e410931f4a0141" - "04f3ce8948c592690cb36a7e6e31534927569f41e45c77fbdb95fdb0e2" - "35717a295b2a281c04619139c453dd9139e951086325b4021bd28da6fc" - "3da86943cd04cfffffffff21fd91e6dd64838eadce9bc0d687af571779" - "c3af4b53bada845485466decf005000000008b48304502205d309b632a" - "816f1f2479b8ca284301fce3fc8f979306722bba4cf28c8fa7ed760221" - "00a3313ee59bbf0e61e29302b2ea597a993a73e93be841f2f7dd3f6e96" - "23572793014104ca3b25750307a483c0cff8efb1413c98b4e18765485d" - "94eac147f21f507169997a72aeb50f0b01ed6b468c9554143125f63c0b" - "2b229a597e695718b86d711607ffffffff4dd6c46e3e68fc05243b6f89" - "4e2f3e4d2782d1c8a3d17b9a2c5e78912bbaf433010000008b48304502" - "2100f3f1d6c0e76a00e89a5278fa14eae8de93454174199f72c98aef54" - "dabb2074d10220172dc0d6958efa8dc18c621f2d56fd0484ecb04e3d65" - "e2284268916c1c324ade014104daeab6af397baf06fcc91b6c04a28ffb" - "2cd80312e954a2a55f7c864094608032ef657e8296dfc2f86ecf28fe32" - "925c0f648d3ef16b96934a34b29dd871ff057affffffff52aabc7fdee0" - "588d63fff3fc00e20b00a87cffa6a37aad7168741b1f0e77bdff000000" - "008b4830450220274a5a9577f0786fb17d373b6d1995a6cff752320bf4" - "e3d5ee41b15fdd86ef2a022100fd1d0c0f43d05e479430c3c9bd5481f5" - "be929a7f2b82d14292ef192653c3b94c014104988828ea7144ff00def9" - "988e88aad98b1f22d01000fc4b52d1003c9b32de0534cff71c0a045098" - "bb28f699bcc9db13533c0998eb2133085d37cac65b5bd74c1fffffffff" - "5aab5581c575a56a803ef5d8698dc34014261d196ada1da3300731380d" - "7e672d000000008b48304502201615a116a2e057f3f8f2a63340d0387b" - "820a6f9d1067758f4f6cdec76c04951a022100a1ada9e03a0d9603743d" - "41347ffd992642544d6dfdc51121b7bf622e12cb1547014104830b3b13" - "9b0a8707f348840b0fa358164869fbc0c423a2785cb0a18ff73517f887" - "8cefd643e86e682eb93d07216f76993c9e980592de07c507c3a2659eff" - "f6aaffffffff74d7cd370a334aabee3b8328b67a63f1768583405e73eb" - "d291dd86189983a61c000000008a4730440220432ff0f90a3f5372a8dc" - "4b3d2e7165e110efa16b40784ea4d5ab866b736712ed02201a759b630d" - "562f4dbc4a7d2fca406ad54e235564346bc360dec26d6e72bb60fa0141" - "04ca0bb72303672885ad953279309694c71574d50d2c6e1f5f1e7ab1b0" - "bf3df67e5439b3f6e2501dda539812e414991ebaa547631374ce2bda9d" - "d7211099886827ffffffff8941eb7fe688ff48357981c4756906e331c0" - "7f332429be5b2cfc7a945644f80f000000008a47304402204e074678b5" - "25927a8f459d1dc93ad738ce02c0ea77c50a749647032eda662eee0220" - "3c1a98e9b116dc2fe9e7325a5696ad8e3ad26d2d10503e6e3209f4f5f4" - "0f6e34014104f9baa86b46b04f2a2dc4270903bcca9444026b7f184d87" - "9f089462e332f260f6de8e05f08285a4a3c716220d9ec0f7e9fde354c7" - "79f32f00dd4d4cd8d4a1ac28ffffffff948be2e03f9537a35319e8cee2" - "b7b9ed2769ca59734e5d4b42951ef7ba3bb4a8000000008b4830450221" - "00fcf1deaaa047e8f1b8b8ac783760ea5a0af15adbb9eed1c1748942a5" - "7510e27402204c278b12c4991480dc714af66395f5d8f42e411423bf27" - "b024f62267c45a0d37014104cfd128c3f322b59c21d452d5c4c7b4d1a3" - "237a2d31c6f4033b32db78c19cb342ffc151acc272a02a13ba985a523b" - "655689d8118f502b8a7fb8df5fc7bd8458ceffffffffb2cae04cf238b4" - "d47c614da5ce9948f089a02f1ff3e22cda6b9f9196bda8721501000000" - "8b483045022017a74373ef19756e275f2d987925a8bd22cfaf1c93eede" - "7db63102dc73b7cfd0022100fd2f040d7d3ccd1a2188923f8c7560253a" - "a2a910d8e9cd06a8a05fc69c12f6ce014104daeab6af397baf06fcc91b" - "6c04a28ffb2cd80312e954a2a55f7c864094608032ef657e8296dfc2f8" - "6ecf28fe32925c0f648d3ef16b96934a34b29dd871ff057affffffffbe" - "75c9f1faf79fb885018e3353ada4a7fb5c989bc26f031c5aee69d1eaba" - "d957000000008a4730440220360d35b0a98dd086d242f8976aa704b808" - "541ad78ef5cba45e4eb73c1efbaaa7022025a3d981b9e0836a2493698b" - "802bf7f0eae72de0252c7eac059ad14b1b27001b014104d969f0d45619" - "6f4c16945bc71fc460615002d1554fc8d774288aa8fc5c9eca1622a3dc" - "b1e6a8314c94844e516c0543316b2e75d465c8fc2f0d0d20334e53f7db" - "ffffffffbcbad95cac8bbe08538f7b12c240e96866358dfa819de6e940" - "bb60ce7fae5c3d000000008b483045022100d916990888d0d26246b8d6" - "cb944b1e8ebd6a4cdb2965ff7e4430883eef02c4e102205693b96aa1d6" - "e9335f3cb2986b7f3a03357959e989570d03a56ccdb18a8d7e6d014104" - "0840a410ed7afdc3e3fe3923ae388413bbd8a9089062403e0adce62212" - "bccfe1f80400530121cfdba5b52f3e16397f1959b14a09089778afd5ff" - "41f8a856980fffffffff02005039278c0400001976a914f1c87a5e8ff7" - "d14e74b858089bf771c94b1b6db488ac00203d88792d00001976a914dc" - "df2f9892bfa1cb086530354eab3ba078a2f0") + data = read_test_data("invalid_tx.txt") self.assertRaises(Exception, Transaction, data) def test_unknown_scripts(self): - with open(os.path.join(dir_path, "scripts_invalid.txt")) as f: - data = a2b_hex(f.read().strip()) - + data = read_test_data("scripts_invalid.txt") tx = Transaction(data) for output in tx.outputs: diff --git a/blockchain_parser/tests/utils.py b/blockchain_parser/tests/utils.py new file mode 100644 index 0000000..9ab34b1 --- /dev/null +++ b/blockchain_parser/tests/utils.py @@ -0,0 +1,9 @@ +import os +from binascii import a2b_hex + +dir_path = os.path.dirname(os.path.realpath(__file__)) + + +def read_test_data(filename): + with open(os.path.join(dir_path, "data/", filename)) as f: + return a2b_hex(f.read().strip()) \ No newline at end of file From 7a9e15c236b10d2a6dff5e696801c0641af72628 Mon Sep 17 00:00:00 2001 From: Antoine Le Calvez Date: Wed, 23 Sep 2020 10:34:29 +0200 Subject: [PATCH 04/26] Fix bugs in script type detection --- blockchain_parser/script.py | 10 ++- blockchain_parser/tests/test_script.py | 112 ++++++++++++++++++++++--- 2 files changed, 108 insertions(+), 14 deletions(-) diff --git a/blockchain_parser/script.py b/blockchain_parser/script.py index 23901d8..d63ebae 100644 --- a/blockchain_parser/script.py +++ b/blockchain_parser/script.py @@ -70,7 +70,10 @@ def operations(self): """ if self._operations is None: # Some coinbase scripts are garbage, they could not be valid - self._operations = list(self.script) + try: + self._operations = list(self.script) + except CScriptInvalidError: + self._operations = [] return self._operations @@ -80,7 +83,7 @@ def value(self): if self._value is None: parts = [] try: - for operation in self.operations: + for operation in list(self.script): if isinstance(operation, bytes): parts.append(b2a_hex(operation).decode("ascii")) else: @@ -138,4 +141,5 @@ def is_multisig(self): def is_unknown(self): return not self.is_pubkeyhash() and not self.is_pubkey() \ and not self.is_p2sh() and not self.is_multisig() \ - and not self.is_return() + and not self.is_return() and not self.is_p2wpkh() \ + and not self.is_p2wsh() diff --git a/blockchain_parser/tests/test_script.py b/blockchain_parser/tests/test_script.py index 8b235bb..1badaa1 100644 --- a/blockchain_parser/tests/test_script.py +++ b/blockchain_parser/tests/test_script.py @@ -11,25 +11,115 @@ import unittest from binascii import a2b_hex -from blockchain_parser.script import Script, is_public_key +from blockchain_parser.script import Script class TestScript(unittest.TestCase): - def test_from_hex(self): + def test_op_return_script(self): case1 = "6a" script = Script.from_hex(a2b_hex(case1)) self.assertEqual("OP_RETURN", script.value) + self.assertFalse(script.is_pubkey()) + self.assertFalse(script.is_multisig()) + self.assertFalse(script.is_p2sh()) + self.assertFalse(script.is_p2wpkh()) + self.assertFalse(script.is_p2wsh()) + self.assertFalse(script.is_pubkeyhash()) + self.assertFalse(script.is_unknown()) + self.assertTrue(script.is_return()) - def test_invalid_script(self): + def test_unknown_script(self): case = "40" script = Script.from_hex(a2b_hex(case)) self.assertEqual("INVALID_SCRIPT", script.value) + self.assertFalse(script.is_pubkey()) + self.assertFalse(script.is_multisig()) + self.assertFalse(script.is_p2sh()) + self.assertFalse(script.is_p2wpkh()) + self.assertFalse(script.is_p2wsh()) + self.assertFalse(script.is_pubkeyhash()) + self.assertTrue(script.is_unknown()) + self.assertFalse(script.is_return()) - def test_is_public_key(self): - case1 = "010000000000000017a91471c5c3727fac8dbace94bd38cf8ac16a034a7" \ - "94787" - self.assertFalse(is_public_key(a2b_hex(case1))) - self.assertFalse(is_public_key(None)) - case3 = "02c0993f639534d348e1dca30566491e6cb11c14afa13ec244c05396a98" \ - "39aeb17" - self.assertTrue(is_public_key(a2b_hex(case3))) + case = "" + script = Script.from_hex(a2b_hex(case)) + self.assertEqual("", script.value) + self.assertFalse(script.is_pubkey()) + self.assertFalse(script.is_multisig()) + self.assertFalse(script.is_p2sh()) + self.assertFalse(script.is_p2wpkh()) + self.assertFalse(script.is_p2wsh()) + self.assertFalse(script.is_pubkeyhash()) + self.assertTrue(script.is_unknown()) + self.assertFalse(script.is_return()) + + def test_multisig_script(self): + case = "514104cc71eb30d653c0c3163990c47b976f3fb3f37cccdcbedb169a1dfef58bbfbfaff7d8a473e7e2e6d317b87bafe8bde97e3cf8f065dec022b51d11fcdd0d348ac4410461cbdcc5409fb4b4d42b51d33381354d80e550078cb532a34bfa2fcfdeb7d76519aecc62770f5b0e4ef8551946d8a540911abe3e7854a26f39f58b25c15342af52ae" + script = Script.from_hex(a2b_hex(case)) + self.assertFalse(script.is_pubkey()) + self.assertTrue(script.is_multisig()) + self.assertFalse(script.is_p2sh()) + self.assertFalse(script.is_p2wpkh()) + self.assertFalse(script.is_p2wsh()) + self.assertFalse(script.is_pubkeyhash()) + self.assertFalse(script.is_unknown()) + self.assertFalse(script.is_return()) + + def test_p2sh_script(self): + case = "a91428ad3e63dcae36e5010527578e2eef0e9eeaf3e487" + script = Script.from_hex(a2b_hex(case)) + self.assertFalse(script.is_pubkey()) + self.assertFalse(script.is_multisig()) + self.assertTrue(script.is_p2sh()) + self.assertFalse(script.is_p2wpkh()) + self.assertFalse(script.is_p2wsh()) + self.assertFalse(script.is_pubkeyhash()) + self.assertFalse(script.is_unknown()) + self.assertFalse(script.is_return()) + + def test_p2wpkh_script(self): + case = "0014c958269b5b6469b6e4b87de1062028ad3bb83cc2" + script = Script.from_hex(a2b_hex(case)) + self.assertFalse(script.is_pubkey()) + self.assertFalse(script.is_multisig()) + self.assertFalse(script.is_p2sh()) + self.assertTrue(script.is_p2wpkh()) + self.assertFalse(script.is_p2wsh()) + self.assertFalse(script.is_pubkeyhash()) + self.assertFalse(script.is_unknown()) + self.assertFalse(script.is_return()) + + def test_p2wsh_script(self): + case = "0020701a8d401c84fb13e6baf169d59684e17abd9fa216c8cc5b9fc63d622ff8c58d" + script = Script.from_hex(a2b_hex(case)) + self.assertFalse(script.is_p2sh()) + self.assertFalse(script.is_multisig()) + self.assertFalse(script.is_p2sh()) + self.assertFalse(script.is_p2wpkh()) + self.assertTrue(script.is_p2wsh()) + self.assertFalse(script.is_pubkeyhash()) + self.assertFalse(script.is_unknown()) + self.assertFalse(script.is_return()) + + def test_pubkeyhash_script(self): + case = "76a914e9629ef6f5b82564a9b2ecae6c288c56fb33710888ac" + script = Script.from_hex(a2b_hex(case)) + self.assertFalse(script.is_pubkey()) + self.assertFalse(script.is_multisig()) + self.assertFalse(script.is_p2sh()) + self.assertFalse(script.is_p2wpkh()) + self.assertFalse(script.is_p2wsh()) + self.assertTrue(script.is_pubkeyhash()) + self.assertFalse(script.is_unknown()) + self.assertFalse(script.is_return()) + + def test_pubkey_script(self): + script = Script.from_hex(a2b_hex("4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac")) + self.assertTrue(script.is_pubkey()) + self.assertFalse(script.is_multisig()) + self.assertFalse(script.is_p2sh()) + self.assertFalse(script.is_p2wpkh()) + self.assertFalse(script.is_p2wsh()) + self.assertFalse(script.is_pubkeyhash()) + self.assertFalse(script.is_unknown()) + self.assertFalse(script.is_return()) From ac3e5fe8bb0146f823358d3599c3055a817ae168 Mon Sep 17 00:00:00 2001 From: Erick Date: Wed, 11 Nov 2020 19:41:50 -0500 Subject: [PATCH 05/26] Adding some transaction specific functionality to library, cleaning up typos --- blockchain_parser/blockchain.py | 41 +++++++++++++++++++++++++-- blockchain_parser/index.py | 17 ++++++++++- blockchain_parser/output.py | 4 +-- blockchain_parser/tests/test_index.py | 15 ++++++++++ 4 files changed, 71 insertions(+), 6 deletions(-) diff --git a/blockchain_parser/blockchain.py b/blockchain_parser/blockchain.py index 8a29bd1..7a12241 100644 --- a/blockchain_parser/blockchain.py +++ b/blockchain_parser/blockchain.py @@ -16,6 +16,11 @@ import stat import plyvel +from blockchain_parser.transaction import Transaction +from blockchain_parser.index import DBTransactionIndex +from blockchain_parser import utils +from binascii import unhexlify +from binascii import hexlify from .block import Block from .index import DBBlockIndex from .utils import format_hash @@ -146,7 +151,6 @@ def _index_confirmed(self, chain_indexes, num_confirmations=6): if len(chain) == num_confirmations: return first_block.hash in chain - def get_ordered_blocks(self, index, start=0, end=None, cache=None): """Yields the blocks contained in the .blk files as per the heigt extract from the leveldb index present at path @@ -168,8 +172,8 @@ def get_ordered_blocks(self, index, start=0, end=None, cache=None): with open(cache, 'wb') as f: pickle.dump(blockIndexes, f) - # remove small forks that may have occured while the node was live. - # Occassionally a node will receive two different solutions to a block + # remove small forks that may have occurred while the node was live. + # Occasionally a node will receive two different solutions to a block # at the same time. The Leveldb index saves both, not pruning the # block that leads to a shorter chain once the fork is settled without # "-reindex"ing the bitcoind block data. This leads to at least two @@ -217,3 +221,34 @@ def get_ordered_blocks(self, index, start=0, end=None, cache=None): break blkFile = os.path.join(self.path, "blk%05d.dat" % blkIdx.file) yield Block(get_block(blkFile, blkIdx.data_pos), blkIdx.height) + + def get_transaction(self, txid, db): + """Yields the transaction contained in the .blk files as a python object, + similar to https://developer.bitcoin.org/reference/rpc/getrawtransaction.html + """ + + byte_arr = bytearray.fromhex(txid) + byte_arr.reverse() + tx_hash = hexlify(b't').decode('utf-8') + hexlify(byte_arr).decode('utf-8') + + tx_hash_fmtd = unhexlify(tx_hash) + raw_hex = db.get(tx_hash_fmtd) + + tx_idx = DBTransactionIndex(utils.format_hash(tx_hash_fmtd), raw_hex) + blk_file = os.path.join(self.path, "blk%05d.dat" % tx_idx.blockfile_no) + raw_hex = get_block(blk_file, tx_idx.file_offset) + + offset = tx_idx.block_offset + + transaction_data = raw_hex[80:] + # Try from 1024 (1KiB) -> 1073741824 (1GiB) slice widths + for j in range(0, 20): + try: + offset_e = offset + (1024 * 2 ** j) + transaction = Transaction.from_hex( + transaction_data[offset:offset_e]) + return transaction + except: + continue + + return None diff --git a/blockchain_parser/index.py b/blockchain_parser/index.py index 8334500..f23fc48 100644 --- a/blockchain_parser/index.py +++ b/blockchain_parser/index.py @@ -51,7 +51,7 @@ def __init__(self, blk_hash, raw_hex): self.undo_pos, i = _read_varint(raw_hex[pos:]) pos += i - assert(pos + 80 == len(raw_hex)) + assert (pos + 80 == len(raw_hex)) self.version, p, m, time, bits, self.nonce = unpack( " Date: Wed, 11 Nov 2020 19:44:53 -0500 Subject: [PATCH 06/26] Adding some transaction specific functionality to library, cleaning up typos --- blockchain_parser/blockchain.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/blockchain_parser/blockchain.py b/blockchain_parser/blockchain.py index 7a12241..39048b5 100644 --- a/blockchain_parser/blockchain.py +++ b/blockchain_parser/blockchain.py @@ -223,13 +223,15 @@ def get_ordered_blocks(self, index, start=0, end=None, cache=None): yield Block(get_block(blkFile, blkIdx.data_pos), blkIdx.height) def get_transaction(self, txid, db): - """Yields the transaction contained in the .blk files as a python object, - similar to https://developer.bitcoin.org/reference/rpc/getrawtransaction.html + """Yields the transaction contained in the .blk files as a python + object, similar to + https://developer.bitcoin.org/reference/rpc/getrawtransaction.html """ byte_arr = bytearray.fromhex(txid) byte_arr.reverse() - tx_hash = hexlify(b't').decode('utf-8') + hexlify(byte_arr).decode('utf-8') + tx_hash = hexlify(b't').decode('utf-8') + \ + hexlify(byte_arr).decode('utf-8') tx_hash_fmtd = unhexlify(tx_hash) raw_hex = db.get(tx_hash_fmtd) From 8aaa95241c81845925b6f038f4ea55aa07616ac3 Mon Sep 17 00:00:00 2001 From: Erick Date: Wed, 11 Nov 2020 19:47:36 -0500 Subject: [PATCH 07/26] Adding some transaction specific functionality to library, cleaning up typos --- blockchain_parser/blockchain.py | 2 +- blockchain_parser/index.py | 6 ++++-- blockchain_parser/tests/test_index.py | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/blockchain_parser/blockchain.py b/blockchain_parser/blockchain.py index 39048b5..b0a72de 100644 --- a/blockchain_parser/blockchain.py +++ b/blockchain_parser/blockchain.py @@ -231,7 +231,7 @@ def get_transaction(self, txid, db): byte_arr = bytearray.fromhex(txid) byte_arr.reverse() tx_hash = hexlify(b't').decode('utf-8') + \ - hexlify(byte_arr).decode('utf-8') + hexlify(byte_arr).decode('utf-8') tx_hash_fmtd = unhexlify(tx_hash) raw_hex = db.get(tx_hash_fmtd) diff --git a/blockchain_parser/index.py b/blockchain_parser/index.py index f23fc48..aaaedc0 100644 --- a/blockchain_parser/index.py +++ b/blockchain_parser/index.py @@ -75,5 +75,7 @@ def __init__(self, txn_hash, raw_hex): self.block_offset, i = _read_varint(raw_hex[pos:]) def __repr__(self): - return "DBTransactionIndex(%s, blockfile_no=%d, file_offset=%d, block_offset=%d)" \ - % (self.hash, self.blockfile_no, self.file_offset, self.block_offset) + return "DBTransactionIndex(%s, blockfile_no=%d, " \ + "file_offset=%d, block_offset=%d)" \ + % (self.hash, self.blockfile_no, + self.file_offset, self.block_offset) diff --git a/blockchain_parser/tests/test_index.py b/blockchain_parser/tests/test_index.py index 1c80509..63aa17d 100644 --- a/blockchain_parser/tests/test_index.py +++ b/blockchain_parser/tests/test_index.py @@ -35,7 +35,8 @@ def test_from_hex(self): class TestDBTransactionIndex(unittest.TestCase): def test_from_hex(self): - key_str = "70ad7da56decc86b8a58ac53dbde792c9e97552cdaafd37312af7c4d5c7d0cc1" + key_str = "70ad7da56decc86b8a58ac53dbde792c9e97552cdaafd37312af7c4" \ + "d5c7d0cc1" value_str = "9071938b980ba4bf39" value_hex = a2b_hex(value_str) From 436bf045595db97fe862ffe8a37059f9265238cb Mon Sep 17 00:00:00 2001 From: Erick Date: Wed, 11 Nov 2020 19:48:26 -0500 Subject: [PATCH 08/26] Adding some transaction specific functionality to library, cleaning up typos --- blockchain_parser/blockchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blockchain_parser/blockchain.py b/blockchain_parser/blockchain.py index b0a72de..454e7c2 100644 --- a/blockchain_parser/blockchain.py +++ b/blockchain_parser/blockchain.py @@ -250,7 +250,7 @@ def get_transaction(self, txid, db): transaction = Transaction.from_hex( transaction_data[offset:offset_e]) return transaction - except: + except Exception: continue return None From 9a4f98c1a4fce0d7892b7d3c4ea2a4cc9f83c52f Mon Sep 17 00:00:00 2001 From: Antoine Le Calvez Date: Wed, 6 Jan 2021 12:28:39 +0100 Subject: [PATCH 09/26] Update README and gitignore --- .gitignore | 3 +++ README.md | 33 +++++++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 94845ce..55495a6 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,6 @@ target/ # PyCharm .idea/ + +.vscode/ +.venv/ diff --git a/README.md b/README.md index 3f79a48..7a58f62 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # bitcoin-blockchain-parser [![Build Status](https://travis-ci.org/alecalve/python-bitcoin-blockchain-parser.svg?branch=master)](https://travis-ci.org/alecalve/python-bitcoin-blockchain-parser) [![Coverage Status](https://coveralls.io/repos/alecalve/python-bitcoin-blockchain-parser/badge.svg?branch=master&service=github)](https://coveralls.io/github/alecalve/python-bitcoin-blockchain-parser?branch=master) -This Python 3 library provides a parser for the raw data stored by bitcoind. +This Python 3 library provides a parser for the raw data stored by bitcoind. ## Features - Detects outputs types @@ -17,10 +17,10 @@ Below are two basic examples for parsing the blockchain. More examples are avail This blockchain parser parses raw blocks saved in Bitcoin Core's `.blk` file format. Bitcoin Core does not guarantee that these blocks are saved in order. If your application does not require that blocks are parsed in order, the `Blockchain.get_unordered_blocks(...)` method can be used: ```python -import os +import os from blockchain_parser.blockchain import Blockchain -# Instantiate the Blockchain by giving the path to the directory +# Instantiate the Blockchain by giving the path to the directory # containing the .blk files created by bitcoind blockchain = Blockchain(os.path.expanduser('~/.bitcoin/blocks')) for block in blockchain.get_unordered_blocks(): @@ -34,7 +34,7 @@ for block in blockchain.get_unordered_blocks(): If maintaining block order is necessary for your application, you should use the `Blockchain.get_ordered_blocks(...)` method. This method uses Bitcoin Core's LevelDB index to locate ordered block data in it's `.blk` files. ```python -import os +import os from blockchain_parser.blockchain import Blockchain # To get the blocks ordered by height, you need to provide the path of the @@ -52,7 +52,7 @@ for block in blockchain.get_ordered_blocks(os.path.expanduser('~/.bitcoin/blocks print("height=%d block=%s" % (block.height, block.hash)) ``` -Building the LevelDB index can take a while which can make iterative development and debugging challenging. For this reason, `Blockchain.get_ordered_blocks(...)` supports caching the LevelDB index database using [pickle](https://docs.python.org/3.6/library/pickle.html). To use a cache simply pass `cache=filename` to the ordered blocks method. If the cached file does not exist it will be created for faster parsing the next time the method is run. If the cached file already exists it will be used instead of re-parsing the LevelDB database. +Building the LevelDB index can take a while which can make iterative development and debugging challenging. For this reason, `Blockchain.get_ordered_blocks(...)` supports caching the LevelDB index database using [pickle](https://docs.python.org/3.6/library/pickle.html). To use a cache simply pass `cache=filename` to the ordered blocks method. If the cached file does not exist it will be created for faster parsing the next time the method is run. If the cached file already exists it will be used instead of re-parsing the LevelDB database. ```python for block in blockchain.get_ordered_blocks(os.path.expanduser('~/.bitcoin/blocks/index'), cache='index-cache.pickle'): @@ -63,6 +63,14 @@ for block in blockchain.get_ordered_blocks(os.path.expanduser('~/.bitcoin/blocks ## Installing +### Using pip + +``` +pip install blockchain-parser +``` + +### Using source + Requirements : python-bitcoinlib, plyvel, coverage for tests plyvel requires leveldb development libraries for LevelDB >1.2.X @@ -73,12 +81,25 @@ On Linux, install libleveldb-dev sudo apt-get install libleveldb-dev ``` +Install dependencies contained in `requirements.txt`: +``` +pip install -r requirements.txt +``` + Then, just run ``` python setup.py install ``` -## Tests +## Developing + +First, setup a virtualenv and install dependencies: + +``` +virtualenv -p python3 .venv +source .venv/bin/activate +pip install -r requirements.txt +``` Run the test suite by lauching ``` From d028c681cca091bfb3c6c999ff1e506198a1557d Mon Sep 17 00:00:00 2001 From: Antoine Le Calvez Date: Wed, 6 Jan 2021 12:29:24 +0100 Subject: [PATCH 10/26] Put Installing section higher up in README.md --- README.md | 92 +++++++++++++++++++++++++++---------------------------- 1 file changed, 45 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 7a58f62..cad8dc2 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,51 @@ This Python 3 library provides a parser for the raw data stored by bitcoind. - Supports SegWit - Supports ordered block parsing +## Installing + +### Using pip + +``` +pip install blockchain-parser +``` + +### Using source + +Requirements : python-bitcoinlib, plyvel, coverage for tests + +plyvel requires leveldb development libraries for LevelDB >1.2.X + +On Linux, install libleveldb-dev + +``` +sudo apt-get install libleveldb-dev +``` + +Install dependencies contained in `requirements.txt`: +``` +pip install -r requirements.txt +``` + +Then, just run +``` +python setup.py install +``` + +## Developing + +First, setup a virtualenv and install dependencies: + +``` +virtualenv -p python3 .venv +source .venv/bin/activate +pip install -r requirements.txt +``` + +Run the test suite by lauching +``` +./tests.sh +``` + ## Examples Below are two basic examples for parsing the blockchain. More examples are available in the examples directory. @@ -61,50 +106,3 @@ for block in blockchain.get_ordered_blocks(os.path.expanduser('~/.bitcoin/blocks **NOTE**: You must manually/programmatically delete the cache file in order to rebuild the cache. Don't forget to do this each time you would like to re-parse the blockchain with a higher block height than the first time you saved the cache file as the new blocks will not be included in the cache. -## Installing - -### Using pip - -``` -pip install blockchain-parser -``` - -### Using source - -Requirements : python-bitcoinlib, plyvel, coverage for tests - -plyvel requires leveldb development libraries for LevelDB >1.2.X - -On Linux, install libleveldb-dev - -``` -sudo apt-get install libleveldb-dev -``` - -Install dependencies contained in `requirements.txt`: -``` -pip install -r requirements.txt -``` - -Then, just run -``` -python setup.py install -``` - -## Developing - -First, setup a virtualenv and install dependencies: - -``` -virtualenv -p python3 .venv -source .venv/bin/activate -pip install -r requirements.txt -``` - -Run the test suite by lauching -``` -./tests.sh -``` - - - From b1ca91e3c1e12a7eb770cb8de6a8e1e75bb47c11 Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Mon, 11 Jan 2021 00:00:00 +0000 Subject: [PATCH 11/26] Add the blk*.data file path basename to Block.blk_file --- blockchain_parser/block.py | 3 ++- blockchain_parser/blockchain.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/blockchain_parser/block.py b/blockchain_parser/block.py index f00db41..15caf22 100644 --- a/blockchain_parser/block.py +++ b/blockchain_parser/block.py @@ -46,7 +46,7 @@ class Block(object): Represents a Bitcoin block, contains its header and its transactions. """ - def __init__(self, raw_hex, height=None): + def __init__(self, raw_hex, height=None, blk_file=None): self.hex = raw_hex self._hash = None self._transactions = None @@ -54,6 +54,7 @@ def __init__(self, raw_hex, height=None): self._n_transactions = None self.size = len(raw_hex) self.height = height + self.blk_file = blk_file def __repr__(self): return "Block(%s)" % self.hash diff --git a/blockchain_parser/blockchain.py b/blockchain_parser/blockchain.py index 454e7c2..c4d24be 100644 --- a/blockchain_parser/blockchain.py +++ b/blockchain_parser/blockchain.py @@ -94,7 +94,7 @@ def get_unordered_blocks(self): """ for blk_file in get_files(self.path): for raw_block in get_blocks(blk_file): - yield Block(raw_block) + yield Block(raw_block, None, os.path.split(blk_file)[1]) def __getBlockIndexes(self, index): """There is no method of leveldb to close the db (and release the lock). From 1cc53ece7bdb808017221743eb11cc3300e76e7d Mon Sep 17 00:00:00 2001 From: Massimo Santini Date: Wed, 8 Jun 2022 17:51:12 +0200 Subject: [PATCH 12/26] Using bytes.hex instead of binascii.hexlify MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To convert from bytes to an hex string the `bytes.hex` method is much faster than `str(hexlify(…).decode('uft-8'))`. $ python3 -m timeit "b'\x00\x00\x00\x00\x00\x19\xd6h\x9c\x08Z\xe1e\x83\x1e\x93O\xf7c\xaeF\xa2\xa6\xc1r\xb3\xf1\xb6\n\x8c\xe2o'.hex()" 5000000 loops, best of 5: 72.3 nsec per loop $ python3 -m timeit -s "from binascii import hexlify" "hexlify(b'\x00\x00\x00\x00\x00\x19\xd6h\x9c\x08Z\xe1e\x83\x1e\x93O\xf7c\xaeF\xa2\xa6\xc1r\xb3\xf1\xb6\n\x8c\xe2o').decode('utf-8')" 2000000 loops, best of 5: 137 nsec per loop $ python3 -m timeit -s "from binascii import hexlify" "str(hexlify(b'\x00\x00\x00\x00\x00\x19\xd6h\x9c\x08Z\xe1e\x83\x1e\x93O\xf7c\xaeF\xa2\xa6\xc1r\xb3\xf1\xb6\n\x8c\xe2o').decode('utf-8'))" 2000000 loops, best of 5: 194 nsec per loop --- blockchain_parser/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blockchain_parser/utils.py b/blockchain_parser/utils.py index 18ce8d8..0967255 100644 --- a/blockchain_parser/utils.py +++ b/blockchain_parser/utils.py @@ -26,7 +26,7 @@ def double_sha256(data): def format_hash(hash_): - return str(hexlify(hash_[::-1]).decode("utf-8")) + return hash_[::-1].hex() def decode_uint32(data): From c334275239cfe7735f867f909a76d315d0aae963 Mon Sep 17 00:00:00 2001 From: Massimo Santini Date: Thu, 9 Jun 2022 15:26:27 +0200 Subject: [PATCH 13/26] Removed stale import Ditto. --- blockchain_parser/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/blockchain_parser/utils.py b/blockchain_parser/utils.py index 0967255..fd04a89 100644 --- a/blockchain_parser/utils.py +++ b/blockchain_parser/utils.py @@ -9,7 +9,6 @@ # modified, propagated, or distributed except according to the terms contained # in the LICENSE file. -from binascii import hexlify import hashlib import struct From 654615e28e4db20d680a649e8e41ec92a69df01c Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Sun, 28 Jan 2024 21:36:49 +0000 Subject: [PATCH 14/26] Update plyvel==1.5.1 to fix fatal error: longintrepr.h: No such file or directory build --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index bb6ec2b..da34f73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ python-bitcoinlib==0.11.0 -plyvel==1.2.0 +plyvel==1.5.1 coverage==4.0.2 diff --git a/setup.py b/setup.py index 06cb1ee..4f9bd63 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,6 @@ ], install_requires=[ 'python-bitcoinlib==0.11.0', - 'plyvel==1.2.0' + 'plyvel==1.5.1' ] ) From 2b59f05d0dd6be68665383ef3603affe630f4960 Mon Sep 17 00:00:00 2001 From: levushakov Date: Tue, 5 Mar 2024 22:02:19 +0000 Subject: [PATCH 15/26] Issue #29 & #105 solved: get_transaction() returns block header --- blockchain_parser/blockchain.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/blockchain_parser/blockchain.py b/blockchain_parser/blockchain.py index c4d24be..55d8b41 100644 --- a/blockchain_parser/blockchain.py +++ b/blockchain_parser/blockchain.py @@ -24,6 +24,7 @@ from .block import Block from .index import DBBlockIndex from .utils import format_hash +from .block_header import BlockHeader # Constant separating blocks in the .blk files @@ -243,13 +244,15 @@ def get_transaction(self, txid, db): offset = tx_idx.block_offset transaction_data = raw_hex[80:] + block_header_data = raw_hex[:80] # Try from 1024 (1KiB) -> 1073741824 (1GiB) slice widths for j in range(0, 20): try: + block_header = BlockHeader.from_hex(block_header_data) offset_e = offset + (1024 * 2 ** j) transaction = Transaction.from_hex( transaction_data[offset:offset_e]) - return transaction + return [block_header, transaction] except Exception: continue From 05f1f02275fdc2dff7d38a7eae08e8823329912e Mon Sep 17 00:00:00 2001 From: Antoine Le Calvez Date: Sat, 23 Mar 2024 14:41:01 +0100 Subject: [PATCH 16/26] Update coverage to its latest version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index da34f73..62477e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ python-bitcoinlib==0.11.0 plyvel==1.5.1 -coverage==4.0.2 +coverage==7.4.4 \ No newline at end of file From 932468786f21ab56b441660c5b97114588367ee2 Mon Sep 17 00:00:00 2001 From: Antoine Le Calvez Date: Sat, 23 Mar 2024 14:41:05 +0100 Subject: [PATCH 17/26] Make README clearer about LevelDB dependency --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index cad8dc2..5af109c 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,14 @@ This Python 3 library provides a parser for the raw data stored by bitcoind. ## Installing +Whether installing using Pip or from source, plyvel requires leveldb development libraries for LevelDB >1.2.X. + +On Linux, install libleveldb-dev + +``` +sudo apt-get install libleveldb-dev +``` + ### Using pip ``` @@ -20,13 +28,6 @@ pip install blockchain-parser Requirements : python-bitcoinlib, plyvel, coverage for tests -plyvel requires leveldb development libraries for LevelDB >1.2.X - -On Linux, install libleveldb-dev - -``` -sudo apt-get install libleveldb-dev -``` Install dependencies contained in `requirements.txt`: ``` From a1ba6083dc2e1724e3e5d5ad04952d172e975fab Mon Sep 17 00:00:00 2001 From: Antoine Le Calvez Date: Sat, 23 Mar 2024 14:57:07 +0100 Subject: [PATCH 18/26] Use Github actions to build package --- .github/workflows/python-package.yml | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/python-package.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..558b312 --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,40 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Python package + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.6", "3.9", "3.10", "3.11"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + pytest From ef092ac9f074a7b26ba72e4f840f12a44e0e96b9 Mon Sep 17 00:00:00 2001 From: Antoine Le Calvez Date: Sat, 23 Mar 2024 15:09:40 +0100 Subject: [PATCH 19/26] Use ripemd-hash instead of hashlib for ripemd160 --- blockchain_parser/utils.py | 5 ++++- requirements.txt | 3 ++- setup.py | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/blockchain_parser/utils.py b/blockchain_parser/utils.py index fd04a89..407b9b2 100644 --- a/blockchain_parser/utils.py +++ b/blockchain_parser/utils.py @@ -12,10 +12,13 @@ import hashlib import struct +from ripemd import ripemd160 def btc_ripemd160(data): + """Computes ripemd160(sha256(data))""" + h1 = hashlib.sha256(data).digest() - r160 = hashlib.new("ripemd160") + r160 = ripemd160.new() r160.update(h1) return r160.digest() diff --git a/requirements.txt b/requirements.txt index 62477e9..c751a70 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ python-bitcoinlib==0.11.0 plyvel==1.5.1 -coverage==7.4.4 \ No newline at end of file +ripemd-hash==1.0.1 +coverage==7.4.4 diff --git a/setup.py b/setup.py index 4f9bd63..214e177 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ ], install_requires=[ 'python-bitcoinlib==0.11.0', - 'plyvel==1.5.1' + 'plyvel==1.5.1', + 'ripemd-hash==1.0.1' ] ) From a010e1e1275699ca859e5c1f391b03a1f4e06d6d Mon Sep 17 00:00:00 2001 From: Antoine Le Calvez Date: Sat, 23 Mar 2024 15:11:18 +0100 Subject: [PATCH 20/26] Build on specific 3.6 python version --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 558b312..cc52e7e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.6", "3.9", "3.10", "3.11"] + python-version: ["3.6.15", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 From 51b6b50e10be79afa1340d654bceda8265fca107 Mon Sep 17 00:00:00 2001 From: Antoine Le Calvez Date: Sat, 23 Mar 2024 15:13:10 +0100 Subject: [PATCH 21/26] Skip building on python 3.6 --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index cc52e7e..14a4e65 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.6.15", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 From 2b4ef10dc17f2a08823b464643457fd1dc72ef02 Mon Sep 17 00:00:00 2001 From: Antoine Le Calvez Date: Sat, 23 Mar 2024 15:16:55 +0100 Subject: [PATCH 22/26] Remove Travis CI references --- .travis.yml | 22 ---------------------- README.md | 2 +- travis.sh | 17 ----------------- 3 files changed, 1 insertion(+), 40 deletions(-) delete mode 100644 .travis.yml delete mode 100755 travis.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 34f2a4c..0000000 --- a/.travis.yml +++ /dev/null @@ -1,22 +0,0 @@ -sudo: required - -language: - - python - -python: - - "3.6" - -before_install: - - ./travis.sh - -install: - - pip install -r requirements.txt - - pip install coveralls - -script: - - coverage run --append --include='blockchain_parser/*' --omit='*/tests/*' setup.py test - -after_success: - - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then - coveralls; - fi diff --git a/README.md b/README.md index 5af109c..4d70857 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# bitcoin-blockchain-parser [![Build Status](https://travis-ci.org/alecalve/python-bitcoin-blockchain-parser.svg?branch=master)](https://travis-ci.org/alecalve/python-bitcoin-blockchain-parser) [![Coverage Status](https://coveralls.io/repos/alecalve/python-bitcoin-blockchain-parser/badge.svg?branch=master&service=github)](https://coveralls.io/github/alecalve/python-bitcoin-blockchain-parser?branch=master) +# bitcoin-blockchain-parser [![Build Status](https://github.com/alecalve/python-bitcoin-blockchain-parser/actions/workflows/python-package/badge.svg)] [![Coverage Status](https://coveralls.io/repos/alecalve/python-bitcoin-blockchain-parser/badge.svg?branch=master&service=github)](https://coveralls.io/github/alecalve/python-bitcoin-blockchain-parser?branch=master) This Python 3 library provides a parser for the raw data stored by bitcoind. ## Features diff --git a/travis.sh b/travis.sh deleted file mode 100755 index c1f0dca..0000000 --- a/travis.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh - -# based on https://github.com/wbolster/plyvel/blob/fa460e431982e94034fe226faef570ce498c89ac/travis.sh -set -e -u -x - -LEVELDB_VERSION=1.20 - -wget https://github.com/google/leveldb/archive/v${LEVELDB_VERSION}.tar.gz -tar xf v${LEVELDB_VERSION}.tar.gz -cd leveldb-${LEVELDB_VERSION}/ -make - -# based on https://gist.github.com/dustismo/6203329 -sudo scp -r out-static/lib* out-shared/lib* /usr/local/lib/ -cd include/ -sudo scp -r leveldb /usr/local/include/ -sudo ldconfig From fe1c1799989556b555c6af71ef1104f70b025721 Mon Sep 17 00:00:00 2001 From: Antoine Le Calvez Date: Sat, 23 Mar 2024 15:19:32 +0100 Subject: [PATCH 23/26] Fix badge URL --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4d70857..e71784c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# bitcoin-blockchain-parser [![Build Status](https://github.com/alecalve/python-bitcoin-blockchain-parser/actions/workflows/python-package/badge.svg)] [![Coverage Status](https://coveralls.io/repos/alecalve/python-bitcoin-blockchain-parser/badge.svg?branch=master&service=github)](https://coveralls.io/github/alecalve/python-bitcoin-blockchain-parser?branch=master) +# bitcoin-blockchain-parser [![Build Status](https://github.com/alecalve/python-bitcoin-blockchain-parser/actions/workflows/python-package.yml/badge.svg)] [![Coverage Status](https://coveralls.io/repos/alecalve/python-bitcoin-blockchain-parser/badge.svg?branch=master&service=github)](https://coveralls.io/github/alecalve/python-bitcoin-blockchain-parser?branch=master) This Python 3 library provides a parser for the raw data stored by bitcoind. ## Features From 890e6d7724e03419cd45a27f069e2e147079def2 Mon Sep 17 00:00:00 2001 From: Antoine Le Calvez Date: Sat, 23 Mar 2024 15:21:18 +0100 Subject: [PATCH 24/26] Remove brackets --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e71784c..ff333df 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# bitcoin-blockchain-parser [![Build Status](https://github.com/alecalve/python-bitcoin-blockchain-parser/actions/workflows/python-package.yml/badge.svg)] [![Coverage Status](https://coveralls.io/repos/alecalve/python-bitcoin-blockchain-parser/badge.svg?branch=master&service=github)](https://coveralls.io/github/alecalve/python-bitcoin-blockchain-parser?branch=master) +# bitcoin-blockchain-parser ![Build Status](https://github.com/alecalve/python-bitcoin-blockchain-parser/actions/workflows/python-package.yml/badge.svg) [![Coverage Status](https://coveralls.io/repos/alecalve/python-bitcoin-blockchain-parser/badge.svg?branch=master&service=github)](https://coveralls.io/github/alecalve/python-bitcoin-blockchain-parser?branch=master) This Python 3 library provides a parser for the raw data stored by bitcoind. ## Features From 2602f1e947aee5dad3f58de33562e2d8e61303ab Mon Sep 17 00:00:00 2001 From: Antoine Le Calvez Date: Sat, 23 Mar 2024 15:27:10 +0100 Subject: [PATCH 25/26] Use Github actions to publish package --- .github/workflows/publish.yml | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..bdaab28 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,39 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} From c6ffc8a537ba629caa5bf6b7045789e5a4248553 Mon Sep 17 00:00:00 2001 From: Antoine Le Calvez Date: Sat, 23 Mar 2024 15:28:59 +0100 Subject: [PATCH 26/26] Update version to 0.1.6 --- blockchain_parser/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blockchain_parser/__init__.py b/blockchain_parser/__init__.py index 4f76129..75da190 100644 --- a/blockchain_parser/__init__.py +++ b/blockchain_parser/__init__.py @@ -9,4 +9,4 @@ # modified, propagated, or distributed except according to the terms contained # in the LICENSE file. -__version__ = "0.1.5" +__version__ = "0.1.6"