util/sectors2sqlite.py

   1 #!/usr/bin/python3
   2
   3 # Loads block files from sectors folders into map.sqlite database.
   4 # The sectors folder should be safe to remove after this prints "Finished."
   5
   6 import time, os, sys
   7
   8 try:
   9         import sqlite3
  10 except:
  11         exit('You need to have the Python sqlite3 module.')
  12
  13 path = "../world/"
  14
  15 paths = []
  16
  17 # sectors2 gets to try first
  18 if os.path.isdir(path + 'sectors2/'):
  19         paths.append('sectors2')
  20 if os.path.isdir(path + 'sectors/'):
  21         paths.append('sectors')
  22
  23 if not paths:
  24         exit('Could not find sectors folder at ' + path + 'sectors2/ or ' + path + 'sectors/')
  25
  26 def parseSigned12bit(u):
  27         u = int('0x'+u, 16)
  28         return (u if u < 2**11 else u - 2**12)
  29
  30 def parseSigned16bit(u):
  31         u = int('0x'+u, 16)
  32         return (u if u < 2**15 else u - 2**16)
  33
  34 def int64(u):
  35         while u >= 2**63:
  36                 u -= 2**64
  37         while u <= -2**63:
  38                 u += 2**64
  39         return u
  40
  41 # Convert sector folder(s) to integer
  42 def getSectorPos(dirname):
  43         if len(dirname) == 8:
  44                 # Old layout
  45                 x = parseSigned16bit(dirname[:4])
  46                 z = parseSigned16bit(dirname[4:])
  47         elif len(dirname) == 7:
  48                 # New layout
  49                 x = parseSigned12bit(dirname[:3])
  50                 z = parseSigned12bit(dirname[4:])
  51         else:
  52                 print('Terrible sector at ' + dirname)
  53                 return
  54
  55         return x, z
  56
  57 # Convert block file to integer position
  58 def getBlockPos(sectordir, blockfile):
  59         p2d = getSectorPos(sectordir)
  60
  61         if not p2d:
  62                 return
  63
  64         if len(blockfile) != 4:
  65                 print("Invalid block filename: " + blockfile)
  66
  67         y = parseSigned16bit(blockfile)
  68
  69         return p2d[0], y, p2d[1]
  70
  71 # Convert location to integer
  72 def getBlockAsInteger(p):
  73         return int64(p[2]*16777216 + p[1]*4096 + p[0])
  74
  75 # Init
  76
  77 create = False
  78 if not os.path.isfile(path + 'map.sqlite'):
  79         create = True
  80
  81 conn = sqlite3.connect(path + 'map.sqlite')
  82
  83 if not conn:
  84         exit('Could not open database.')
  85
  86 cur = conn.cursor()
  87
  88 if create:
  89         cur.execute("CREATE TABLE IF NOT EXISTS `blocks` (`pos` INT NOT NULL PRIMARY KEY, `data` BLOB);")
  90         conn.commit()
  91         print('Created database at ' + path + 'map.sqlite')
  92
  93 # Crawl the folders
  94
  95 count = 0
  96 t = time.time()
  97 for base in paths:
  98         v = 0
  99         if base == 'sectors':
 100                 v = 1
 101         elif base == 'sectors2':
 102                 v= 2
 103         else:
 104                 print('Ignoring base ' + base)
 105                 continue
 106
 107         for root, dirs, files in os.walk(path + base):
 108                 if files:
 109                         for block in files:
 110                                 pos = getBlockAsInteger(getBlockPos(root[(-8 if v == 1 else -7 if v == 2 else 0):], block))
 111
 112                                 if pos is None:
 113                                         print('Ignoring broken path ' + root + '/' + block)
 114                                         continue
 115
 116                                 f = open(root+'/'+block, 'rb')
 117                                 blob = f.read()
 118                                 f.close()
 119                                 if sys.version_info.major == 2:
 120                                         blob = buffer(blob)
 121                                 else:
 122                                         blob = memoryview(blob)
 123                                 cur.execute('INSERT OR IGNORE INTO `blocks` VALUES(?, ?)', (pos, blob))
 124                                 count += 1
 125
 126                 if(time.time() - t > 3):
 127                         t = time.time()
 128                         print(str(count)+' blocks processed...')
 129
 130 conn.commit()
 131
 132 print('Finished. (' + str(count) + ' blocks)')