From abcde0d1c8ddcd02c52caa360e5917c7d24d608a Mon Sep 17 00:00:00 2001 From: MelKori Date: Mon, 4 Jul 2022 21:59:28 +0300 Subject: [PATCH 1/6] update wagon and clojure --- profiles.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/profiles.clj b/profiles.clj index f2ba097..2d74a94 100644 --- a/profiles.clj +++ b/profiles.clj @@ -3,10 +3,10 @@ :target-path "target/%s" :clean-targets ^{:protect false} [:target-path] - :plugins [[org.apache.maven.wagon/wagon-ssh-external "3.4.3"] - [org.apache.maven.wagon/wagon-http-lightweight "3.4.3"]]} + :plugins [[org.apache.maven.wagon/wagon-ssh-external "3.5.2"] + [org.apache.maven.wagon/wagon-http-lightweight "3.5.2"]]} - :provided {:dependencies [[org.clojure/clojure "1.10.3"]] + :provided {:dependencies [[org.clojure/clojure "1.11.1"]] :source-paths #{"src-clj"} :java-source-paths #{"src-java"} :resource-paths ["resources"] From babc1603ba7fcd5bbc8d593737d52e4389fc383c Mon Sep 17 00:00:00 2001 From: MelKori Date: Fri, 27 Jan 2023 13:18:45 +0200 Subject: [PATCH 2/6] drop wagon plugins from profiles --- profiles.clj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/profiles.clj b/profiles.clj index 2d74a94..b5993a7 100644 --- a/profiles.clj +++ b/profiles.clj @@ -3,8 +3,7 @@ :target-path "target/%s" :clean-targets ^{:protect false} [:target-path] - :plugins [[org.apache.maven.wagon/wagon-ssh-external "3.5.2"] - [org.apache.maven.wagon/wagon-http-lightweight "3.5.2"]]} + :plugins []} :provided {:dependencies [[org.clojure/clojure "1.11.1"]] :source-paths #{"src-clj"} From 33a09238f90bf8daed3e44efb794b2b7d9be4c99 Mon Sep 17 00:00:00 2001 From: MelKori Date: Fri, 3 Feb 2023 14:07:34 +0200 Subject: [PATCH 3/6] extracted CRC32C for some Intel chips; refactored checksum tests --- src-clj/zlib_tiny/core.clj | 6 +++++- test/zlib_tiny/checksum.clj | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src-clj/zlib_tiny/core.clj b/src-clj/zlib_tiny/core.clj index 0592bff..d4372c7 100644 --- a/src-clj/zlib_tiny/core.clj +++ b/src-clj/zlib_tiny/core.clj @@ -6,7 +6,7 @@ DeflaterInputStream Deflater ZipException - CRC32) + CRC32 CRC32C) (zlib_tiny CRC64) (org.apache.commons.io IOUtils) (java.security MessageDigest) @@ -100,6 +100,10 @@ ^Long [^bytes b] (wrap-crc CRC32 b)) +(defn crc32c + ^Long [^bytes b] + (wrap-crc CRC32C b)) + (defn crc64 ^Long [^bytes b] (wrap-crc CRC64 b)) diff --git a/test/zlib_tiny/checksum.clj b/test/zlib_tiny/checksum.clj index c278293..1ed7d8c 100644 --- a/test/zlib_tiny/checksum.clj +++ b/test/zlib_tiny/checksum.clj @@ -6,10 +6,19 @@ (def test-string "test it!") (deftest checksums - (testing "CRC32" - (is (= 3421780262 (crc32 (.getBytes "123456789"))))) - (testing "CRC64" - (is (= -7395533204333446662 (crc64 (.getBytes "123456789")))))) + (let [input "123456789" + bs (.getBytes input)] + (println "Test input string:" input) + (println "Test input bytes:" (hexlify bs)) + (testing "CRC32" + (println "CRC32 checks:") + (is (= 3421780262 (time (crc32 bs))) "CRC32 mismatch")) + (testing "CRC32C" + (println "CRC32C checks:") + (is (= 3808858755 (time (crc32c bs))) "CRC32C mismatch")) + (testing "CRC64" + (println "CRC64 checks:") + (is (= -7395533204333446662 (time (crc64 bs))) "CRC64 mismatch")))) (deftest digests (testing "MD5" From 1fac618e1b4a5372b9c9fa07f8b30d8a58b58ad0 Mon Sep 17 00:00:00 2001 From: MelKori Date: Fri, 3 Feb 2023 14:43:45 +0200 Subject: [PATCH 4/6] added builtin crc32c; extracted adler32 --- src-clj/zlib_tiny/core.clj | 11 ++- src-java/CRC32C.java | 137 ++++++++++++++++++++++++++++++++++++ test/zlib_tiny/checksum.clj | 3 + 3 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 src-java/CRC32C.java diff --git a/src-clj/zlib_tiny/core.clj b/src-clj/zlib_tiny/core.clj index d4372c7..f608eae 100644 --- a/src-clj/zlib_tiny/core.clj +++ b/src-clj/zlib_tiny/core.clj @@ -6,8 +6,9 @@ DeflaterInputStream Deflater ZipException - CRC32 CRC32C) - (zlib_tiny CRC64) + CRC32 + Adler32) + (zlib_tiny CRC32C CRC64) (org.apache.commons.io IOUtils) (java.security MessageDigest) (java.io ByteArrayInputStream @@ -75,7 +76,7 @@ (InflaterInputStream. stream (Inflater. true)))))) (defn deflate - "Returns a deflate'd version of the given byte array." + "Returns deflate'd version of the given byte array." ([b] (when b (IOUtils/toByteArray (DeflaterInputStream. (ByteArrayInputStream. b))))) @@ -104,6 +105,10 @@ ^Long [^bytes b] (wrap-crc CRC32C b)) +(defn adler32 + ^Long [^bytes b] + (wrap-crc Adler32 b)) + (defn crc64 ^Long [^bytes b] (wrap-crc CRC64 b)) diff --git a/src-java/CRC32C.java b/src-java/CRC32C.java new file mode 100644 index 0000000..2bc28ce --- /dev/null +++ b/src-java/CRC32C.java @@ -0,0 +1,137 @@ +package zlib_tiny; + +import java.util.zip.Checksum; + +/** + * This class generates a CRC32C checksum, defined by rfc3720 section B.4. + * The native implementation is not available in Java versions older than Java 9 + * The implementation is ported from Google's cloud package, copyrighted 2011. + * Works well for Intel and ARM chips independently of Java version. + */ +public final class CRC32C implements Checksum { + + private static final long[] CRC_TABLE = { + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, + 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, + 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, + 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, + 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, + 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, + 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, + 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, + 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, + 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, + 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, + 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, + 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, + 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, + 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, + 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, + 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, + 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, + 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, + 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, + 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, + 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, + 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, + 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, + 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, + 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, + 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, + 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, + 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, + 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, + 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, + 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, + 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, + 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, + 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, + 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, + 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, + 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, + 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, + 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, + 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, + 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, + 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, + 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, + 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, + 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, + 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, + 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, + 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, + 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, + 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, + 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, + 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, + 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, + 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, + 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, + 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, + 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, + 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, + 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, + 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, + 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, + 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351 + }; + + private static final long LONG_MASK = 0xffffffffL; + private static final long BYTE_MASK = 0xff; + + private long crc; + + public CRC32C() { + crc = 0; + } + + /** + * Updates the checksum with a new byte. + * @param b the new byte. + */ + @Override + public void update(int b) { + long newCrc = crc ^ LONG_MASK; + newCrc = updateByte((byte) b, newCrc); + crc = newCrc ^ LONG_MASK; + } + + /** + * Updates the checksum with an array of bytes. + * @param bArray the array of bytes. + * @param off the offset into the array where the update should begin. + * @param len the length of data to examine. + */ + @Override + public void update(byte[] bArray, int off, int len) { + long newCrc = crc ^ LONG_MASK; + for (int i = off; i < off + len; i++) { + newCrc = updateByte(bArray[i], newCrc); + } + crc = newCrc ^ LONG_MASK; + } + + /** + * Returns the value of the checksum. + * @return the long representation of the checksum (high bits set to zero). + */ + @Override + public long getValue() { + return crc; + } + + /** + * Resets the crc. + */ + @Override + public void reset() { + crc = 0; + } + + private long updateByte(byte newByte, long crc) { + byte b = (byte) (newByte & BYTE_MASK); + int index = (int) ((crc ^ b) & BYTE_MASK); + return (CRC_TABLE[index] ^ (crc >> 8)) & LONG_MASK; + } +} diff --git a/test/zlib_tiny/checksum.clj b/test/zlib_tiny/checksum.clj index 1ed7d8c..5510422 100644 --- a/test/zlib_tiny/checksum.clj +++ b/test/zlib_tiny/checksum.clj @@ -16,6 +16,9 @@ (testing "CRC32C" (println "CRC32C checks:") (is (= 3808858755 (time (crc32c bs))) "CRC32C mismatch")) + (testing "Adler32" + (println "Adler32 checks:") + (is (= 152961502 (time (adler32 bs))) "Adler32 mismatch")) (testing "CRC64" (println "CRC64 checks:") (is (= -7395533204333446662 (time (crc64 bs))) "CRC64 mismatch")))) From c36da8d5a599f6dc74cbd9d258b76a7bb34e9f07 Mon Sep 17 00:00:00 2001 From: MelKori Date: Fri, 3 Feb 2023 15:19:41 +0200 Subject: [PATCH 5/6] extracted sha-384; updated readme; prepared release --- README.adoc | 41 ++++++++++++++++++++++++++++++++----- project.clj | 2 +- src-clj/zlib_tiny/core.clj | 4 ++++ test/zlib_tiny/checksum.clj | 6 ++++++ 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/README.adoc b/README.adoc index c75b0b5..486b5d0 100644 --- a/README.adoc +++ b/README.adoc @@ -17,10 +17,10 @@ Add the following to your http://github.com/technomancy/leiningen[Leiningen's] ` [source,clojure] ---- -[net.tbt-post/zlib-tiny "0.4.1"] +[net.tbt-post/zlib-tiny "0.5.0"] ---- -CAUTION: From version v0.3.2 and upward the library requires Java class versions 53.0 and newer (J11+), thus if in your needs is to use it with Java 1.8 please switch back to older (0.2.x) version of the library or try to rebuild the library locally. +CAUTION: From version v0.3.2 and upward the library may require Java class versions 53.0 and newer (J11+), thus if in your needs is to use it with some of Java 1.8, and the library build is not compatible with your version of Java, please switch back to older (0.2.x) version of the library (if your project not requires fresh features) or try to rebuild the library locally. We are doing our best to keep the library backward compatible with older Java versions, but unfortunately can not guaranty complete compatibility with wide range of java builds. === Compress @@ -73,6 +73,20 @@ CAUTION: From version v0.3.2 and upward the library requires Java class versions => 3421780262 ---- +[source,clojure] +---- +;; CRC32C example +(crc32c (.getBytes "123456789")) +=> 3808858755 +---- + +[source,clojure] +---- +;; Adler32 example +(adler32 (.getBytes "123456789")) +=> 152961502 +---- + [source,clojure] ---- ;; CRC64 example @@ -92,6 +106,12 @@ $ echo -n 'test it!' | shasum -a 1 $ echo -n 'test it!' | shasum -a 256 9c507d01834b2749d088122a7b3d200957f9b25579b5ce6b490e3b2067ee4f66 - + +$ echo -n 'test it!' | shasum -a 384 +6e5cc5271b2255f2cf4154c3170c5fb09059c79d28d182ac2caa59bd607ea87c09637d8f2f7b400ac80810f13027716a - + +$ echo -n 'test it!' | shasum -a 512 +15353093ef47d2eadefc55d7bc641b6f1150e0b28a609d2368394748091f20b9125e98fe0603b2fbe57f9d65a9b286a8d0dbf70e8f597525051b6f9220e9b61f - ---- [source,clojure] @@ -102,6 +122,8 @@ $ echo -n 'test it!' | shasum -a 256 => "1393ce5dfcf39109a420eb583ecfdeacc28c783a" (-> "test it!" str->bytes sha-256 hexlify) => "9c507d01834b2749d088122a7b3d200957f9b25579b5ce6b490e3b2067ee4f66" +(-> "test it!" str->bytes sha-384 hexlify) +=> "6e5cc5271b2255f2cf4154c3170c5fb09059c79d28d182ac2caa59bd607ea87c09637d8f2f7b400ac80810f13027716a" (-> "test it!" str->bytes sha-512 hexlify) => "15353093ef47d2eadefc55d7bc641b6f1150e0b28a609d2368394748091f20b9125e98fe0603b2fbe57f9d65a9b286a8d0dbf70e8f597525051b6f9220e9b61f" ---- @@ -115,11 +137,20 @@ $ lein test ... lein test zlib-tiny.checksum +Test input string: 123456789 +Test input bytes: 313233343536373839 +CRC32 checks: +"Elapsed time: 0.034417 msecs" +CRC32C checks: +"Elapsed time: 0.037292 msecs" +Adler32 checks: +"Elapsed time: 0.316375 msecs" +CRC64 checks: +"Elapsed time: 0.210833 msecs" lein test zlib-tiny.compress -Ran 3 tests containing 10 assertions. -0 failures, 0 errors. +Ran 3 tests containing 13 assertions. ---- == Manual Build @@ -131,7 +162,7 @@ $ lein install == License -Copyright © 2017-2021 +Copyright © 2017-2023 Distributed under the http://www.apache.org/licenses/LICENSE-2.0[Apache License v 2.0] diff --git a/project.clj b/project.clj index 7fe540f..89fafd4 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject net.tbt-post/zlib-tiny "0.4.1" +(defproject net.tbt-post/zlib-tiny "0.5.0" :description "Tiny Clojure ZLib helper" :url "https://github.com/source-c/zlib-tiny" :license {:name "Apache License v2.0" diff --git a/src-clj/zlib_tiny/core.clj b/src-clj/zlib_tiny/core.clj index f608eae..9d83b1e 100644 --- a/src-clj/zlib_tiny/core.clj +++ b/src-clj/zlib_tiny/core.clj @@ -131,6 +131,10 @@ ^bytes [^bytes b] (wrap-digest "SHA-256" b)) +(defn sha-384 + ^bytes [^bytes b] + (wrap-digest "SHA-384" b)) + (defn sha-512 ^bytes [^bytes b] (wrap-digest "SHA-512" b)) diff --git a/test/zlib_tiny/checksum.clj b/test/zlib_tiny/checksum.clj index 5510422..f2cd94b 100644 --- a/test/zlib_tiny/checksum.clj +++ b/test/zlib_tiny/checksum.clj @@ -41,6 +41,12 @@ str->bytes sha-256 hexlify)))) + (testing "SHA384" + (is (= "6e5cc5271b2255f2cf4154c3170c5fb09059c79d28d182ac2caa59bd607ea87c09637d8f2f7b400ac80810f13027716a" + (-> test-string + str->bytes + sha-384 + hexlify)))) (testing "SHA512" (is (= "15353093ef47d2eadefc55d7bc641b6f1150e0b28a609d2368394748091f20b9125e98fe0603b2fbe57f9d65a9b286a8d0dbf70e8f597525051b6f9220e9b61f" (-> test-string From c8862826be34563f507d426400c76e833dddc418 Mon Sep 17 00:00:00 2001 From: MelKori Date: Fri, 3 Feb 2023 15:20:51 +0200 Subject: [PATCH 6/6] fix readme --- README.adoc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/README.adoc b/README.adoc index 486b5d0..d05fad9 100644 --- a/README.adoc +++ b/README.adoc @@ -82,16 +82,18 @@ CAUTION: From version v0.3.2 and upward the library may require Java class versi [source,clojure] ---- -;; Adler32 example -(adler32 (.getBytes "123456789")) -=> 152961502 +;; CRC64 example +(crc64 (.getBytes "123456789")) +=> -7395533204333446662 ---- +==== Alternatives + [source,clojure] ---- -;; CRC64 example -(crc64 (.getBytes "123456789")) -=> -7395533204333446662 +;; Adler32 example +(adler32 (.getBytes "123456789")) +=> 152961502 ---- ==== Digests