Extended vox format not supported
destrosvet opened this issue · 16 comments
After import *.vox nothing happend and got "Unknown Chunk id nTRN" in console.
@destrosvet I ran into the same issue. It's because the file format of MagicaVoxel has undergone some revisions to allow for a world editor and this importer doesn't know how to handle that data. Additionally the old material data chunks were deprecated and a new MATL chunk is being used. A workaround I used was to google for an older version of MagicaVoxel with a compatible file version.
Apologies for the delayed response.
I didn't know about the format change, so thank you for bringing that to my attention.
Current solution would be as @atomhax suggests; using an old version of MagicaVoxel, until I get around to updating this importer.
@destrosvet, could you please attach a sample data that you can't load?
@atomhax, have you got a file using MATL chuck?
Thanks for your help
About new releases chunks, we have: nTRN, nGRP, nGRP, nSHP, rOBJ, LAYR, MATL
@destrosvet, @atomhax, just to "skip" them and keep loading new VOX files (here there is an example using them chr_sword NEW.zip):
elif name == 'nTRN':
vox.read(s_self)
elif name == 'nGRP':
vox.read(s_self)
elif name == 'nGRP':
vox.read(s_self)
elif name == 'MATL':
vox.read(s_self)
elif name == 'LAYR':
vox.read(s_self)
elif name == 'rOBJ':
vox.read(s_self)
Right before the "throw an error if unknow chunk" part:
else:
# Any other chunk, we don't know how to handle
# This puts us out-of-step
print('Unknown Chunk id {}'.format(name))
return {'CANCELLED'}
Sample data, for example: chr_sword NEW.zip
Parser snippet for new chunks (just some random thoughts to keep track of them):
def read_STRING(vox):
size, = struct.unpack('<i', vox.read(4))
str = vox.read(size)
str = str.decode('utf-8')
return str
def read_DICT(vox):
num_of_key_value_pairs, = struct.unpack('<i', vox.read(4))
if num_of_key_value_pairs > 0:
print("\tkeys:", num_of_key_value_pairs)
for i in range(num_of_key_value_pairs):
str1 = read_STRING(vox)
str2 = read_STRING(vox)
print("\tkey-value: ('{}', '{}')".format(str1, str2))
def read_ROTATION(vox):
# store a row-major rotation in the bits of a byte
print("TODO")
As usual, in main loop:
elif name == 'nTRN':
node_id, = struct.unpack('<i', vox.read(4))
print("nTRN({}):".format(node_id))
read_DICT(vox)
child_node_id, = struct.unpack('<i', vox.read(4))
reserved_id, = struct.unpack('<i', vox.read(4))
layer_id, = struct.unpack('<i', vox.read(4))
num_of_frames, = struct.unpack('<i', vox.read(4))
assert(reserved_id == -1) # sanity check
assert(num_of_frames == 1) # sanity check
for i in range(num_of_frames):
# for each frame
read_DICT(vox)
elif name == 'nGRP':
node_id, = struct.unpack('<i', vox.read(4))
print("nGRP({}):".format(node_id))
read_DICT(vox)
children, = struct.unpack('<i', vox.read(4))
for i in range(children):
# for each child
child_node_id, = struct.unpack('<i', vox.read(4))
print("\tchild_node_id:", str(child_node_id))
elif name == 'nSHP':
node_id, = struct.unpack('<i', vox.read(4))
print("nSHP({}):".format(node_id))
read_DICT(vox)
children, = struct.unpack('<i', vox.read(4))
assert(children== 1) # sanity check
for i in range(children):
# for each child
child_node_id, = struct.unpack('<i', vox.read(4))
print("\tchild_node_id:", str(child_node_id))
read_DICT(vox)
elif name == 'MATL':
# material
matl_id, = struct.unpack('<i', vox.read(4))
print("MATL({}):".format(matl_id))
read_DICT(vox)
elif name == 'LAYR':
layr_id, = struct.unpack('<i', vox.read(4))
print("LAYR({}):".format(layr_id))
read_DICT(vox)
reserved_id, = struct.unpack('<i', vox.read(4))
assert(reserved_id == -1) # sanity check
elif name == 'rOBJ':
#robj_id, = struct.unpack('<i', vox.read(4))
print("rOBJ:")
read_DICT(vox)
else:
...................
p.s. is it possible to change the issue title?
Fantastic work so far 👏
I have updated issue title.
Always for the "random thoughts" series... about rOBJ chunk:
I think MagicaVoxel uses them to store some additional data we can skip in Blender, e.g.
View menu options?
Parsing chunk 'rOBJ':
rOBJ:
keys: 3
key-value: ('_type', '_ground')
key-value: ('_color', '80 80 80')
key-value: ('_enable', '1')
Reading at dec: 39677 hex: 9AFD
Parsing chunk 'rOBJ':
rOBJ:
keys: 3
key-value: ('_type', '_bg')
key-value: ('_color', '0 0 0')
key-value: ('_enable', '0')
Reading at dec: 39744 hex: 9B40
Parsing chunk 'rOBJ':
rOBJ:
keys: 4
key-value: ('_type', '_edge')
key-value: ('_color', '0 0 0')
key-value: ('_width', '0.2')
key-value: ('_enable', '0')
Reading at dec: 39830 hex: 9B96
Parsing chunk 'rOBJ':
rOBJ:
keys: 6
key-value: ('_type', '_grid')
key-value: ('_color', '0 0 0')
key-value: ('_spacing', '1')
key-value: ('_width', '0.05')
key-value: ('_display', '0')
key-value: ('_enable', '0')
Reading at dec: 39951 hex: 9C0F
Rendering mode: path tracking renderer, camera, bloom and light info?
Reading at dec: 38948 hex: 9824
Parsing chunk 'rOBJ':
rOBJ:
keys: 5
key-value: ('_type', '_inf')
key-value: ('_i', '0.6')
key-value: ('_k', '255 255 255')
key-value: ('_angle', '50 50')
key-value: ('_area', '0.07')
Reading at dec: 39051 hex: 988B
Parsing chunk 'rOBJ':
rOBJ:
keys: 3
key-value: ('_type', '_uni')
key-value: ('_i', '0.7')
key-value: ('_k', '255 255 255')
Reading at dec: 39118 hex: 98CE
Parsing chunk 'rOBJ':
rOBJ:
keys: 3
key-value: ('_type', '_rayleigh')
key-value: ('_d', '0.4')
key-value: ('_k', '77 153 255')
Reading at dec: 39189 hex: 9915
Parsing chunk 'rOBJ':
rOBJ:
keys: 4
key-value: ('_type', '_mie')
key-value: ('_d', '0.4')
key-value: ('_k', '255 255 255')
key-value: ('_g', '0.78')
Reading at dec: 39270 hex: 9966
Parsing chunk 'rOBJ':
rOBJ:
keys: 4
key-value: ('_type', '_fog')
key-value: ('_d', '0.2')
key-value: ('_k', '255 255 255')
key-value: ('_enable', '0')
Reading at dec: 39353 hex: 99B9
Parsing chunk 'rOBJ':
rOBJ:
keys: 9
key-value: ('_type', '_len')
key-value: ('_fov', '45')
key-value: ('_dof', '0.25')
key-value: ('_exp', '0')
key-value: ('_vig', '0')
key-value: ('_sg', '0')
key-value: ('_gam', '2.2')
key-value: ('_blade_n', '0')
key-value: ('_blade_r', '0')
Reading at dec: 39503 hex: 9A4F
Parsing chunk 'rOBJ':
rOBJ:
keys: 5
key-value: ('_type', '_bloom')
key-value: ('_mix', '0.5')
key-value: ('_scale', '0')
key-value: ('_aspect', '0')
key-value: ('_threshold', '1')
Reading at dec: 39603 hex: 9AB3
Credits go to Interface · MagicaVoxel User Reference Manual
Oh wow.
That's some great detective work.
I'm happy for all that detail to be skipped, though I suppose there is the possibility of including them in the future.
The ground color, lighting, and camera position all jump out to me as something potentially useful. These would be fairly easy to replicate in Blender too, I would imagine.
Yeah, but my doubt is:
Is it better to work with overall scene setup, animations, camera, lights, global parameters (like fov, fog, ...) directly in Blender, after importing objects I mean, or in MagicaVoxel editor importing those settings?
Anyhow, now we briefly know info about other chunks and we can handle them (or just skip) if needed. My personal opinion about new chunks (in some sort of "priority" order for next implementations):
- parse
MATT(you got it already, so we can start from your old Materials branch) - parse
MATL(both "matter" chunks should share a common data-structure for later handling) - handle
nTRN(i.e. roto-translations and maybe keyframing) - parse
nGRPandnSHPfor grouping - skip
rOBJ
Thanks again!
Absolutely, I don't think they should be a priority in any sense.
More that it's just cool to know that we could import them, should we ever deem that a nice feature.
Starting from your Materials branch, I'm going to implement MATT and MATL here.
Still immature for a pull request, but if you have time to review and give me some hints... especially, as I am not a Python programmer, I'm afraid of making mistakes.
I am using your "named tuple" structure for both chunks (the material_spec class).
My first goal: load and store Matter properties first. Use them and create Materials in Blender, then.
Have a nice weekend. Bye bye
To supply "default values" to namedtuples, I defined a new class:
# http://mmabrouk.github.io/python/2014/05/23/namedtuples-with-default-optional-arguments/
class material_spec(namedtuple('material_spec', [ # Material chunks, both MATT and MATL
'type_', # Matter type: _diffuse, _metal, _glass, _emit
'weight', #
'Plastic', #
'Roughness', #
'Specular', #
'IOR', # Index of refraction
'Attenuation', #
'Power', #
'Glow', #
'isTotalPower' # Unused?
])):
def __new__(cls,
# TODO: Default values?
type_ = "_diffuse",
weight = 1.0,
Plastic = 0.0,
Roughness = 0.5,
Specular = 0.5,
IOR = 1.45,
Attenuation = 0.0,
Power = 0.0,
Glow = 0.0,
isTotalPower = True
):
return super(material_spec, cls).__new__(cls,
type_,
weight,
Plastic,
Roughness,
Specular,
IOR,
Attenuation,
Power,
Glow,
isTotalPower
)
That linked page is quite dated.
Python 3.7 (which Blender 2.80 uses) added an additional argument to namedtuple, specifically to handle default arguments.
https://docs.python.org/3.7/library/collections.html#collections.namedtuple
material_spec = namedtuple('material_spec', [ # Material chunks, both MATT and MATL
'type_', # Matter type: _diffuse, _metal, _glass, _emit
'weight', #
'Plastic', #
'Roughness', #
'Specular', #
'IOR', # Index of refraction
'Attenuation', #
'Power', #
'Glow', #
'isTotalPower' # Unused?
],
defaults = [ # TODO: Default values?
"_diffuse", # type_
1.0, # weight
0.0, # Plastic
0.5, # Roughness
0.5, # Specular
1.45, # IOR
0.0, # Attenuation
0.0, # Power
0.0, # Glow
True # isTotalPower
]
)
Expanded out here, for the sake of commenting.
Oh, thanks, very nice. But I read that tuples are "immutable", so I cannot modify values later.
Here it is the full updated script: https://github.com/wizardgsz/MagicaVoxel-VOX-importer/blob/MATT_MATL/io_scene_vox.py
Having spec as "named tuple", I have to use the _replace method.
I read about recordtype, a mutable type, but it seems Blender does not support it.
spec = material_spec(type_=material_types[matt_type], weight=weight)
..
# For each property, for example Power:
value = ...read it from file (for MATT), or from DICT structure (from MATL)
spec = spec._replace(Power = value)
That is:
def read_MATT(vox, material_specs):
matt_id, matt_type, weight = struct.unpack('<iif', vox.read(12))
print("MATT({}):".format(matt_id))
material_types = ["_diffuse", "_metal", "_glass", "_emit"]
prop_bits, = struct.unpack('<i', vox.read(4))
binary = bin(prop_bits)
spec = material_spec(type_=material_types[matt_type], weight=weight)
if prop_bits & 1:
value, = struct.unpack('<f', vox.read(4))
spec = spec._replace(Plastic = value)
if prop_bits & 2:
value, = struct.unpack('<f', vox.read(4))
spec = spec._replace(Roughness = value)
if prop_bits & 4:
value, = struct.unpack('<f', vox.read(4))
spec = spec._replace(Specular = value)
if prop_bits & 8:
value, = struct.unpack('<f', vox.read(4))
spec = spec._replace(IOR = value)
if prop_bits & 16:
value, = struct.unpack('<f', vox.read(4))
spec = spec._replace(Attenuation = value)
if prop_bits & 32:
value, = struct.unpack('<f', vox.read(4))
spec = spec._replace(Power = value)
if prop_bits & 64:
value, = struct.unpack('<f', vox.read(4))
spec = spec._replace(Glow = value)
if prop_bits & 128:
value, struct.unpack('<f', vox.read(4))
# isTotalPower is never supplied by this
material_specs.update({matt_id: spec})
return spec
def read_MATL(vox, material_specs):
matl_id, = struct.unpack('<i', vox.read(4))
print("MATL({}):".format(matl_id))
# TODO: Error handling for missing properties?
properties = read_DICT(vox)
matl_type = properties['_type']
weight = float(properties['_weight'])
spec = material_spec(type_=matl_type, weight=weight)
if '_plastic' in properties:
value = float(properties['_plastic'])
spec = spec._replace(Plastic = value)
if '_rough' in properties:
value = float(properties['_rough'])
spec = spec._replace(Roughness = value)
if '_spec' in properties:
value = float(properties['_spec'])
spec = spec._replace(Specular = value)
if '_ior' in properties:
value = float(properties['_ior'])
spec = spec._replace(IOR = value)
if '_att' in properties:
value = float(properties['_att'])
spec = spec._replace(Attenuation = value)
Power = 1.0
if '_glow' in properties:
value = float(properties['_glow'])
spec = spec._replace(Glow = value)
# isTotalPower is never supplied by this
material_specs.update({matl_id: spec})
return spec
Hmm. Honestly, it might just be easier to use a bare dictionary, and forgo the namedtuple plan altogether.
For all the work that's going in to work around namedtuple, is it really giving any tangible benefit?
About MATT chunk and isTotalPower flag (property bit(7) set to 1): I just discovered that sometimes, but not always, MagicaVoxels adds 4 bytes for it!
It is better to save current offset and skip entirely the chunk at the end:
def read_MATT(vox, material_specs, s_self):
# Get the current position of the file
offset = vox.tell()
...
...
vox.seek(offset)
vox.read(s_self)
material_specs.update({matt_id: spec})
return spec