How to save and reload a very large amount of data?

[amorphia]

I thought I'd share how I solved this in the end, as it wasn't simple.

I tried to roll my own saving routines based on modified versions of this sort of code, but whether I gradually concatenate the values into a string and then save it to file, or append the values to file one by one, the results are really painfully slow.

So I looked around for more serialization libraries, and tried three or four. They all crashed, either when saving the data, or when loading it again. In the end, I took what seemed to me the most stable one, http://piratery.net/dump/, and used it by first splitting my table into 10 tables and saving them, and then loading 10 tables and putting them back together:

Code: Select all

dump = require 'dump'

--the code which saves

	local regSize = #drawReg.reg -- this is the table which I need to save
	local sliceSize = math.floor(regSize / 10)
	for i = 0,9 do
		local slice = table.slice(drawReg.reg,i*sliceSize+1,(i+1)*sliceSize)
		local str = dump.tostring(slice)
		love.filesystem.write("replay"..i..".lua",str)
	end

-- the code which loads

	for i = 0,9 do
		local s = love.filesystem.read("replay"..i..".txt",str)
		local script = 'return '..s
		local slice = loadstring(script)() -- note the double set of parenthesis
		table.conc(replayReg.reg,slice) -- replayReg.reg is the table I need to load into
	end

-- functions I found on the internet for slicing tables and putting them back together again

function table.slice(tbl, first, last, step)
  local sliced = {}
  for i = first or 1, last or #tbl, step or 1 do
    sliced[#sliced+1] = tbl[i]
  end
  return sliced
end

function table.conc(t1,t2) -- concatenate
    for i=1,#t2 do
        t1[#t1+1] = t2[i]
    end
    return t1
end