Add support for mia and sha256.

nih-at · May 10, 2024 · 83a08be · 83a08be
1 parent 32c2aff
commit 83a08be
Show file tree

Hide file tree

Showing 67 changed files with 1,160 additions and 679 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -62,6 +62,7 @@ add_dependencies(distcheck dist)
 
 check_function_exists(MD5Init HAVE_MD5INIT)
 check_function_exists(SHA1Init HAVE_SHA1INIT)
+check_function_exists(SHA256Init HAVE_SHA256INIT)
 check_function_exists(fnmatch HAVE_FNMATCH)
 check_symbol_exists(_stricmp string.h HAVE__STRICMP)
 check_symbol_exists(strcasecmp strings.h HAVE_STRCASECMP)

diff --git a/cmake-config.h.in b/cmake-config.h.in
@@ -11,6 +11,7 @@
 #cmakedefine HAVE_FNMATCH
 #cmakedefine HAVE_MD5INIT
 #cmakedefine HAVE_SHA1INIT
+#cmakedefine HAVE_SHA256INIT
 #cmakedefine HAVE_STRCASECMP
 #cmakedefine HAVE__STRICMP
 

diff --git a/regress/1-4-as-2-48.ckmamedb-dump b/regress/1-4-as-2-48.ckmamedb-dump
@@ -1,5 +1,5 @@
 >>> table archive (archive_id, name, mtime, size, file_type)
 1|2-48.zip|1422359238|114|0
 >>> table detector (detector_id, name, version)
->>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, detector_id)
-1|0|04.rom|1047617702|0|4|3632233996|<098f6bcd4621d373cade4e832627b4f6>|<a94a8fe5ccb19ba61c4c0873d391e987982fbbd3>|0
+>>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, sha256, detector_id)
+1|0|04.rom|1047617702|0|4|3632233996|<098f6bcd4621d373cade4e832627b4f6>|<a94a8fe5ccb19ba61c4c0873d391e987982fbbd3>|<9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08>|0
diff --git a/regress/1-4-ok-1-8-ok.ckmamedb-dump b/regress/1-4-ok-1-8-ok.ckmamedb-dump
@@ -2,6 +2,6 @@
 1|1-4|1422359238|0|0
 2|1-8|1422359888|0|0
 >>> table detector (detector_id, name, version)
->>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, detector_id)
-1|0|04.rom|1047617702|0|4|3632233996|<098f6bcd4621d373cade4e832627b4f6>|<a94a8fe5ccb19ba61c4c0873d391e987982fbbd3>|0
-2|0|08.rom|1047652618|0|8|911640957|<095ca6fcc1279865662b553147eb8f6d>|<111bb8b7549e3386a996845405b02164f17c7b37>|0
+>>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, sha256, detector_id)
+1|0|04.rom|1047617702|0|4|3632233996|<098f6bcd4621d373cade4e832627b4f6>|<a94a8fe5ccb19ba61c4c0873d391e987982fbbd3>|<9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08>|0
+2|0|08.rom|1047652618|0|8|911640957|<095ca6fcc1279865662b553147eb8f6d>|<111bb8b7549e3386a996845405b02164f17c7b37>|<75423ebdb12042cecfe1e6de984bda7e74163fea1770dcf31280437993c46e8d>|0
diff --git a/regress/1-4-ok-with-mtime.ckmamedb-dump b/regress/1-4-ok-with-mtime.ckmamedb-dump
@@ -1,5 +1,5 @@
 >>> table archive (archive_id, name, mtime, size, file_type)
 1|1-4.zip|1419260288|114|0
 >>> table detector (detector_id, name, version)
->>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, detector_id)
-1|0|04.rom|1047617702|0|4|3632233996|<098f6bcd4621d373cade4e832627b4f6>|<a94a8fe5ccb19ba61c4c0873d391e987982fbbd3>|0
+>>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, sha256, detector_id)
+1|0|04.rom|1047617702|0|4|3632233996|<098f6bcd4621d373cade4e832627b4f6>|<a94a8fe5ccb19ba61c4c0873d391e987982fbbd3>|<9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08>|0
diff --git a/regress/1-4-ok.ckmamedb-dump b/regress/1-4-ok.ckmamedb-dump
@@ -1,5 +1,5 @@
 >>> table archive (archive_id, name, mtime, size, file_type)
 1|1-4|1422359238|0|0
 >>> table detector (detector_id, name, version)
->>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, detector_id)
-1|0|04.rom|1047617702|0|4|3632233996|<098f6bcd4621d373cade4e832627b4f6>|<a94a8fe5ccb19ba61c4c0873d391e987982fbbd3>|0
+>>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, sha256, detector_id)
+1|0|04.rom|1047617702|0|4|3632233996|<098f6bcd4621d373cade4e832627b4f6>|<a94a8fe5ccb19ba61c4c0873d391e987982fbbd3>|<9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08>|0
diff --git a/regress/1-8-ok-v2-migrated.ckmamedb-dump b/regress/1-8-ok-v2-migrated.ckmamedb-dump
@@ -0,0 +1,5 @@
+>>> table archive (archive_id, name, mtime, size, file_type)
+2|1-8.zip|1715349226|118|0
+>>> table detector (detector_id, name, version)
+>>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, detector_id, sha256)
+2|0|08.rom|1047652618|0|8|911640957|<095ca6fcc1279865662b553147eb8f6d>|<111bb8b7549e3386a996845405b02164f17c7b37>|0|<75423ebdb12042cecfe1e6de984bda7e74163fea1770dcf31280437993c46e8d>
diff --git a/regress/1-8-ok.ckmamedb-dump b/regress/1-8-ok.ckmamedb-dump
@@ -1,5 +1,5 @@
 >>> table archive (archive_id, name, mtime, size, file_type)
 1|1-8|0|0|0
 >>> table detector (detector_id, name, version)
->>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, detector_id)
-1|0|08.rom|1047649018|0|8|911640957|<095ca6fcc1279865662b553147eb8f6d>|<111bb8b7549e3386a996845405b02164f17c7b37>|0
+>>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, sha256, detector_id)
+1|0|08.rom|1047649018|0|8|911640957|<095ca6fcc1279865662b553147eb8f6d>|<111bb8b7549e3386a996845405b02164f17c7b37>|<75423ebdb12042cecfe1e6de984bda7e74163fea1770dcf31280437993c46e8d>|<e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855>|0
diff --git a/regress/2-4c-ok.ckmamedb-dump b/regress/2-4c-ok.ckmamedb-dump
@@ -1,6 +1,6 @@
 >>> table archive (archive_id, name, mtime, size, file_type)
 1|2-4c.zip|1419271671|214|0
 >>> table detector (detector_id, name, version)
->>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, detector_id)
-1|0|04.rom|1047617702|0|4|3632233996|<098f6bcd4621d373cade4e832627b4f6>|<a94a8fe5ccb19ba61c4c0873d391e987982fbbd3>|0
-1|1|0c.rom|1047837840|0|12|103008562|<b60c52bf4849067f0b57c8bd30985466>|<2f2d205d5451d3256cf1c693982b40101e9989bf>|0
+>>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, sha256, detector_id)
+1|0|04.rom|1047617702|0|4|3632233996|<098f6bcd4621d373cade4e832627b4f6>|<a94a8fe5ccb19ba61c4c0873d391e987982fbbd3>|<9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08>|0
+1|1|0c.rom|1047837840|0|12|103008562|<b60c52bf4849067f0b57c8bd30985466>|<2f2d205d5451d3256cf1c693982b40101e9989bf>|<d407ad901895723f32d31e6515f5284beb4c01e70e56720d65099da4771f3193>|0
diff --git a/regress/2-4c-reverse.ckmamedb-dump b/regress/2-4c-reverse.ckmamedb-dump
@@ -1,6 +1,6 @@
 >>> table archive (archive_id, name, mtime, size, file_type)
 1|2-4c.zip|1419271683|214|0
 >>> table detector (detector_id, name, version)
->>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, detector_id)
-1|0|0c.rom|1047837840|0|12|103008562|<b60c52bf4849067f0b57c8bd30985466>|<2f2d205d5451d3256cf1c693982b40101e9989bf>|0
-1|1|04.rom|1047617702|0|4|3632233996|<098f6bcd4621d373cade4e832627b4f6>|<a94a8fe5ccb19ba61c4c0873d391e987982fbbd3>|0
+>>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, sha256, detector_id)
+1|0|0c.rom|1047837840|0|12|103008562|<b60c52bf4849067f0b57c8bd30985466>|<2f2d205d5451d3256cf1c693982b40101e9989bf>|<d407ad901895723f32d31e6515f5284beb4c01e70e56720d65099da4771f3193>|0
+1|1|04.rom|1047617702|0|4|3632233996|<098f6bcd4621d373cade4e832627b4f6>|<a94a8fe5ccb19ba61c4c0873d391e987982fbbd3>|<9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08>|0
diff --git a/regress/ckmamedb-migrate-v2.zip.test b/regress/ckmamedb-migrate-v2.zip.test
@@ -4,7 +4,8 @@ return 0
 arguments -Fvc -je extra 1-4
 file mame.db mame.db
 file extra/1-8.zip 1-8-ok.zip
-file extra/.ckmame.db 1-8-ok-v2-ckmame.db <empty.ckmamedb>
+set-modification-time extra/1-8.zip 1715349226
+file extra/.ckmame.db 1-8-ok-v2-ckmame.db 1-8-ok-v2-migrated.ckmamedb-dump
 stdout
 In game 1-4:
 game 1-4                                     : not a single file found

diff --git a/regress/compare-ckmamedb b/regress/compare-ckmamedb
@@ -11,7 +11,7 @@ import nihtest.CompareArrays
 
 table_archive = ">>> table archive (archive_id, name, mtime, size, file_type)"
 table_detector = ">>> table detector (detector_id, name, version)"
-table_file = ">>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, detector_id)"
+table_file = ">>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, sha256, detector_id)"
 
 
 class Table(enum.Enum):
@@ -20,7 +20,7 @@ class Table(enum.Enum):
     FILE = enum.auto()
 
 
-re_atrribute = re.compile("^([^=]+)=(.*)")
+re_attribute = re.compile("^([^=]+)=(.*)")
 
 status_names = {
     "ok": 0,
@@ -66,14 +66,15 @@ class Index:
         return None
 
 class File:
-    def __init__(self, name, index=0, mtime="<null>", status=0, size="-1", crc="<null>", md5="<null>", sha1="<null>", detector_id=0):
+    def __init__(self, name, index=0, mtime="<null>", status=0, size="-1", crc="<null>", md5="<null>", sha1="<null>", sha256="<null>", detector_id=0):
         self.name = name
         self.index = index
         self.status = status
         self.size = size
         self.crc = crc
         self.md5 = md5
         self.sha1 = sha1
+        self.sha256 = sha256
         self.mtime = mtime
         self.detector_id = detector_id
 
@@ -85,6 +86,7 @@ class File:
                 and self.crc == other.crc
                 and self.md5 == other.md5
                 and self.sha1 == other.sha1
+                and self.sha256 == other.sha256
                 and int(self.detector_id) == int(other.detector_id))
 
     def __lt__(self, other):
@@ -120,12 +122,16 @@ class File:
             return True
         if self.sha1 > other.sha1:
             return False
+        if self.sha256 < other.sha256:
+            return True
+        if self.sha256 > other.sha256:
+            return False
         if int(self.detector_id) < int(other.detector_id):
             return True
         return False
 
     def row(self, archive_id):
-        return f"{archive_id}|{self.index}|{self.name}|{self.mtime}|{self.status}|{self.size}|{self.crc}|{self.md5}|{self.sha1}|{self.detector_id}"
+        return f"{archive_id}|{self.index}|{self.name}|{self.mtime}|{self.status}|{self.size}|{self.crc}|{self.md5}|{self.sha1}|{self.sha256}|{self.detector_id}"
 
 
 class Archive:
@@ -261,17 +267,17 @@ class CkmameDBComparator:
         archive = arguments[0]
         # TODO: strip .zip for variant unzipped
         file = arguments[1]
-        hashes = {"crc", "md5", "sha1"}
+        hashes = {"crc", "md5", "sha1", "sha256"}
         for spec in arguments[2].split(","):
             if spec == "cheap":
                 hashes = {"crc"} # TODO: {} for unzipped
             elif spec == "none":
                 hashes = {}
             elif spec == "all":
-                hashes = {"crc", "md5", "sha1"}
-            elif spec == "crc" or spec == "md5" or spec == "sha1":
+                hashes = {"crc", "md5", "sha1", "sha256"}
+            elif spec == "crc" or spec == "md5" or spec == "sha1" or spec == "sha256":
                 hashes.add(spec)
-            elif spec == "-crc" or spec == "-md5" or spec == "-sha1":
+            elif spec == "-crc" or spec == "-md5" or spec == "-sha1" or spec == "-sha256":
                 hashes.remove(spec[1:])
             else:
                 print(f"invalid hash spec '{spec}'")
@@ -348,7 +354,7 @@ class CkmameDBComparator:
             # TODO error
             0 == 0
         else:
-            archive.files.append(File(fields[2], fields[1], fields[3], fields[4], fields[5], fields[6], fields[7], fields[8], fields[9]))
+            archive.files.append(File(fields[2], fields[1], fields[3], fields[4], fields[5], fields[6], fields[7], fields[8], fields[9], fields[10]))
 
     def read_directory(self):
         arguments = ["-o", "/dev/stdout", "--runtest"]
@@ -386,7 +392,7 @@ class CkmameDBComparator:
 
             attributes = {}
             for attribute in words[1:]:
-                match = re.match(re_atrribute, attribute)
+                match = re.match(re_attribute, attribute)
                 # TODO: error if not matched
                 attributes[match.group(1)] = match.group(2)
 
@@ -425,6 +431,8 @@ class CkmameDBComparator:
                     file.md5 = "<" + attributes["md5"] + ">"
                 if "sha1" in attributes and "sha1" in hashes:
                     file.sha1 = "<" + attributes["sha1"] + ">"
+                if "sha256" in attributes and "sha256" in hashes:
+                    file.sha256 = "<" + attributes["sha256"] + ">"
                 if "status" in attributes:
                     if attributes["status"] in status_names:
                         file.status = status_names[attributes["status"]]
@@ -447,7 +455,7 @@ class CkmameDBComparator:
                         db.add_detector(Detector(detector_id, name, version))
                     else:
                         detector_id = detector_ids[detector_hash]
-                    detector_file = File("", file.index, size="<???>", crc="<???>", md5="<???>", sha1="<???>")
+                    detector_file = File("", file.index, size="<???>", crc="<???>", md5="<???>", sha1="<???>", sha256="<???>")
                     detector_file.detector_id = detector_id
                     detector_file.status = 0
                     if got_archive is not None:
@@ -459,6 +467,7 @@ class CkmameDBComparator:
                             detector_file.crc = got_file.crc
                             detector_file.md5 = got_file.md5
                             detector_file.sha1 = got_file.sha1
+                            detector_file.sha256 = got_file.sha256
                     archive.add_detector_hash(detector_file)
 
         self.expected_db = db
@@ -468,7 +477,7 @@ class CkmameDBComparator:
             if key in self.hash_exceptions:
                 return self.hash_exceptions[key]
         else:
-            return {"crc", "md5", "sha1"}
+            return {"crc", "md5", "sha1", "sha256"}
 
     def get_status_override(self, archive_name, file_name):
         for key in (f"{archive_name}/{file_name}", f"{archive_name}/*"):

diff --git a/regress/empty.ckmamedb-dump b/regress/empty.ckmamedb-dump
@@ -1,3 +1,3 @@
 >>> table archive (archive_id, name, mtime, size, file_type)
 >>> table detector (detector_id, name, version)
->>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, detector_id)
+>>> table file (archive_id, file_idx, name, mtime, status, size, crc, md5, sha1, sha256, detector_id)
diff --git a/regress/fix-dump b/regress/fix-dump
@@ -0,0 +1,39 @@
+#!/usr/bin/env perl
+
+use strict;
+
+
+for my $file (@ARGV) {
+    process($file);
+}
+
+
+sub process {
+    my $file = shift;
+
+    my @lines;
+    my $in_files = 0;
+    my $fh;
+    open($fh, "<", $file) or die "can't read $file: $!";
+    while (my $line = <$fh>) {
+        chomp $line;
+        if ($line =~ m/^>>> table file.*sha256/) {
+            return;
+        }
+        if ($line =~ m/^>>> table file.*merge.*sha1/) {
+            $line = ">>> table file (game_id, file_type, file_idx, name, merge, status, location, size, crc, md5, sha1, sha256, missing)";
+            $in_files = 1;
+        }
+        elsif ($line =~ m/^>>> table /) {
+            $in_files = 0;
+        }
+        elsif ($in_files) {
+            $line .= "|<null>|0";
+        }
+        push @lines, $line . "\n";
+    }
+    close($fh);
+    open($fh, ">", $file) or die "can't write $file: $!";
+    print $fh join("", @lines);
+    close($fh);
+}