Question

When your data consists of many numerical and matrices, each of which is relatively independent, relational databases offer little benefit, and it is more efficient to use HDF5 (Hierarchical Data Format) for storage. For example, your data may come from a simulation which generates a 3D matrix and a list of count data at every iteration.

import h5py

f = h5py.File('simulation.h5')

for i in range(10): # iterations in simulation
    xs = np.random.random((100,100,100))
    ys = np.random.randint(0,100,(i+1)*10)
    group = f.create_group('Iteration%03d' % i)
    group.create_dataset('xs', data=xs)
    group.create_dataset('ys', data=ys)

f.keys()

[u'Iteration000',
 u'Iteration001',
 u'Iteration002',
 u'Iteration003',
 u'Iteration004',
 u'Iteration005',
 u'Iteration006',
 u'Iteration007',
 u'Iteration008',
 u'Iteration009']

f['Iteration008'].keys()

[u'xs', u'ys']

g8 = f['Iteration008']
print g8['xs'][2:5,2:5,2:5]
print g8['ys'][-10:]

[[[ 0.0367  0.2883  0.5562]
  [ 0.9494  0.5614  0.1159]
  [ 0.8887  0.7396  0.891 ]]

 [[ 0.7552  0.1539  0.216 ]
  [ 0.6671  0.4682  0.9107]
  [ 0.5565  0.5443  0.1665]]

 [[ 0.3972  0.1205  0.9487]
  [ 0.7874  0.3466  0.2818]
  [ 0.1248  0.0161  0.6898]]]
[37 69  5 15 10 44 20 73 74 24]