Files to database converter

diff --git a/util/sectors2sqlite.py b/util/sectors2sqlite.py
new file mode 100755 (executable)
index 0000000..7b002bf
--- /dev/null
+++ b/util/sectors2sqlite.py
@@ -0,0 +1,135 @@
+#!/usr/bin/python3
+
+# Loads block files from sectors folders into map.sqlite database.
+# The sectors folder should be safe to remove after this prints "Finished."
+
+import time, os
+
+try:
+       import sqlite3
+except:
+       exit('You need to have the Python sqlite3 module.')
+
+path = "../world/"
+
+paths = []
+
+# sectors2 gets to try first
+if os.path.isdir(path + 'sectors2/'):
+       paths.append('sectors2')
+if os.path.isdir(path + 'sectors/'):
+       paths.append('sectors')
+
+if not paths:
+       exit('Could not find sectors folder at ' + path + 'sectors2/ or ' + path + 'sectors/')
+
+def uint(u):
+       u = int('0x'+u, 16)
+       return (u if u < 2**11 else u - 2**12)
+
+def int64(u):
+       while u >= 2**63:
+               u -= 2**64
+       while u <= -2**63:
+               u += 2**64
+       return u
+
+# Convert sector folder(s) to integer
+def getSectorPos(dirname):
+       if len(dirname) == 8:
+               # Old layout
+               x = uint(dirname[:4])
+               z = uint(dirname[4:])
+       elif len(dirname) == 7:
+               # New layout
+               x = uint(dirname[:3])
+               z = uint(dirname[4:])
+       else:
+               print('Terrible sector at ' + dirname)
+               return
+       
+       return x, z
+
+# Convert block file to integer position
+def getBlockPos(sectordir, blockfile):
+       p2d = getSectorPos(sectordir)
+       
+       if not p2d:
+               return
+       
+       if len(blockfile) != 4:
+               print("Invalid block filename: " + blockfile)
+       
+       y = uint(blockfile)
+       
+       return p2d[0], y, p2d[1]
+
+# Convert location to integer
+def getBlockAsInteger(p):
+       return int64(p[2]*16777216 + p[1]*4096 + p[0])
+
+# Init
+
+create = False
+if not os.path.isfile(path + 'map.sqlite'):
+       create = True
+
+conn = sqlite3.connect(path + 'map.sqlite')
+
+if not conn:
+       exit('Could not open database.')
+
+cur = conn.cursor()
+
+if create:
+       cur.execute("CREATE TABLE IF NOT EXISTS `blocks` (`pos` INT NOT NULL PRIMARY KEY, `data` BLOB);")
+       conn.commit()
+       print('Created database at ' + path + 'map.sqlite')
+
+# Crawl the folders
+
+count = 0
+t = time.time()
+for base in paths:
+       v = 0
+       if base == 'sectors':
+               v = 1
+       elif base == 'sectors2':
+               v= 2
+       else:
+               print('Ignoring base ' + base)
+               continue
+       
+       for root, dirs, files in os.walk(path + base):
+               if files:
+                       for block in files:
+                               pos = getBlockAsInteger(getBlockPos(root[(-8 if v == 1 else -7 if v == 2 else 0):], block))
+                               
+                               if pos is None:
+                                       print('Ignoring broken path ' + root + '/' + block)
+                                       continue
+                               
+                               f = open(root+'/'+block, 'rb')
+                               cur.execute('INSERT OR IGNORE INTO `blocks` VALUES(?, ?)', (pos, f.read()))
+                               f.close()
+                               count += 1
+               
+               if(time.time() - t > 3):
+                       t = time.time()
+                       print(str(count)+' blocks processed...')
+
+conn.commit()
+
+print('Finished. (' + str(count) + ' blocks)')
+
+
+
+
+
+
+
+
+
+
+
+