gen_esp32part.py

#!/usr/bin/env python
#
# ESP32 partition table generation tool
#
# Converts partition tables to/from CSV and binary formats.
#
# See https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/partition-tables.html
# for explanation of partition table structure and uses.
#
# Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ importprint_function, division
importargparse
importos
importre
importstruct
importsys
importhashlib
importbinascii

MAX_PARTITION_LENGTH=0xC00# 3K for partition data (96 entries) leaves 1K in a 4K sector for signature
MD5_PARTITION_BEGIN=b"\xEB\xEB"+b"\xFF"*14# The first 2 bytes are like magic numbers for MD5 sum
PARTITION_TABLE_SIZE=0x1000# Size of partition table

__version__='1.2'

APP_TYPE=0x00
DATA_TYPE=0x01

TYPES= {
"app" : APP_TYPE,
"data" : DATA_TYPE,
}

# Keep this map in sync with esp_partition_subtype_t enum in esp_partition.h
SUBTYPES= {
APP_TYPE : {
"factory" : 0x00,
"test" : 0x20,
 },
DATA_TYPE : {
"ota" : 0x00,
"phy" : 0x01,
"nvs" : 0x02,
"coredump" : 0x03,
"esphttpd" : 0x80,
"fat" : 0x81,
"spiffs" : 0x82,
 },
}

quiet=False
md5sum=True
offset_part_table=0

defstatus(msg):
""" Print status message to stderr """
ifnotquiet:
critical(msg)

defcritical(msg):
""" Print critical message to stderr """
sys.stderr.write(msg)
sys.stderr.write('\n')

classPartitionTable(list):
def__init__(self):
super(PartitionTable, self).__init__(self)

@classmethod
deffrom_csv(cls, csv_contents):
res=PartitionTable()
lines=csv_contents.splitlines()

defexpand_vars(f):
f=os.path.expandvars(f)
m=re.match(r'(?<!\\)\$([A-Za-z_][A-Za-z0-9_]*)', f)
ifm:
raiseInputError("unknown variable '%s'"%m.group(1))
returnf

forline_noinrange(len(lines)):
line=expand_vars(lines[line_no]).strip()
ifline.startswith("#") orlen(line) ==0:
continue
try:
res.append(PartitionDefinition.from_csv(line))
exceptInputErrorase:
raiseInputError("Error at line %d: %s"% (line_no+1, e))
exceptException:
critical("Unexpected error parsing line %d: %s"% (line_no+1, line))
raise

# fix up missing offsets & negative sizes
last_end=offset_part_table+PARTITION_TABLE_SIZE# first offset after partition table
foreinres:
ifoffset_part_table!=0ande.offsetisnotNoneande.offset<last_end:
critical("WARNING: 0x%x address in the partition table is below 0x%x"% (e.offset, last_end))
e.offset=None
ife.offsetisNone:
pad_to=0x10000ife.type==APP_TYPEelse4
iflast_end%pad_to!=0:
last_end+=pad_to- (last_end%pad_to)
e.offset=last_end
ife.size<0:
e.size=-e.size-e.offset
last_end=e.offset+e.size

returnres

def__getitem__(self, item):
""" Allow partition table access via name as well as by
 numeric index. """
ifisinstance(item, str):
forxinself:
ifx.name==item:
returnx
raiseValueError("No partition entry named '%s'"%item)
else:
returnsuper(PartitionTable, self).__getitem__(item)

deffind_by_type(self, ptype, subtype):
""" Return a partition by type & subtype, returns
 None if not found """
# convert ptype & subtypes names (if supplied this way) to integer values
try:
ptype=TYPES[ptype]
exceptKeyError:
try:
ptypes=int(ptype, 0)
exceptTypeError:
pass
try:
subtype=SUBTYPES[int(ptype)][subtype]
exceptKeyError:
try:
ptypes=int(ptype, 0)
exceptTypeError:
pass

forpinself:
ifp.type==ptypeandp.subtype==subtype:
returnp
returnNone

deffind_by_name(self, name):
forpinself:
ifp.name==name:
returnp
returnNone

defverify(self):
# verify each partition individually
forpinself:
p.verify()
# check for overlaps
last=None
forpinsorted(self, key=lambdax:x.offset):
ifp.offset<offset_part_table+PARTITION_TABLE_SIZE:
raiseInputError("Partition offset 0x%x is below 0x%x"% (p.offset, offset_part_table+PARTITION_TABLE_SIZE))
iflastisnotNoneandp.offset<last.offset+last.size:
raiseInputError("Partition at 0x%x overlaps 0x%x-0x%x"% (p.offset, last.offset, last.offset+last.size-1))
last=p

defflash_size(self):
""" Return the size that partitions will occupy in flash
 (ie the offset the last partition ends at)
 """
try:
last=sorted(self, reverse=True)[0]
exceptIndexError:
return0# empty table!
returnlast.offset+last.size

@classmethod
deffrom_binary(cls, b):
md5=hashlib.md5();
result=cls()
foroinrange(0,len(b),32):
data=b[o:o+32]
iflen(data) !=32:
raiseInputError("Partition table length must be a multiple of 32 bytes")
ifdata==b'\xFF'*32:
returnresult# got end marker
ifmd5sumanddata[:2] ==MD5_PARTITION_BEGIN[:2]: #check only the magic number part
ifdata[16:] ==md5.digest():
continue# the next iteration will check for the end marker
else:
raiseInputError("MD5 checksums don't match! (computed: 0x%s, parsed: 0x%s)"% (md5.hexdigest(), binascii.hexlify(data[16:])))
else:
md5.update(data)
result.append(PartitionDefinition.from_binary(data))
raiseInputError("Partition table is missing an end-of-table marker")

defto_binary(self):
result=b"".join(e.to_binary() foreinself)
ifmd5sum:
result+=MD5_PARTITION_BEGIN+hashlib.md5(result).digest()
iflen(result )>=MAX_PARTITION_LENGTH:
raiseInputError("Binary partition table length (%d) longer than max"%len(result))
result+=b"\xFF"* (MAX_PARTITION_LENGTH-len(result)) # pad the sector, for signing
returnresult

defto_csv(self, simple_formatting=False):
rows= [ "# Espressif ESP32 Partition Table",
"# Name, Type, SubType, Offset, Size, Flags" ]
rows+= [ x.to_csv(simple_formatting) forxinself ]
return"\n".join(rows) +"\n"

classPartitionDefinition(object):
MAGIC_BYTES=b"\xAA\x50"

ALIGNMENT= {
APP_TYPE : 0x10000,
DATA_TYPE : 0x04,
 }

# dictionary maps flag name (as used in CSV flags list, property name)
# to bit set in flags words in binary format
FLAGS= {
"encrypted" : 0
 }

# add subtypes for the 16 OTA slot values ("ota_XX, etc.")
forota_slotinrange(16):
SUBTYPES[TYPES["app"]]["ota_%d"%ota_slot] =0x10+ota_slot

def__init__(self):
self.name=""
self.type=None
self.subtype=None
self.offset=None
self.size=None
self.encrypted=False

@classmethod
deffrom_csv(cls, line):
""" Parse a line from the CSV """
line_w_defaults=line+",,,,"# lazy way to support default fields
fields= [ f.strip() forfinline_w_defaults.split(",") ]

res=PartitionDefinition()
res.name=fields[0]
res.type=res.parse_type(fields[1])
res.subtype=res.parse_subtype(fields[2])
res.offset=res.parse_address(fields[3])
res.size=res.parse_address(fields[4])
ifres.sizeisNone:
raiseInputError("Size field can't be empty")

flags=fields[5].split(":")
forflaginflags:
ifflagincls.FLAGS:
setattr(res, flag, True)
eliflen(flag) >0:
raiseInputError("CSV flag column contains unknown flag '%s'"% (flag))

returnres

def__eq__(self, other):
returnself.name==other.nameandself.type==other.type \
andself.subtype==other.subtypeandself.offset==other.offset \
andself.size==other.size

def__repr__(self):
defmaybe_hex(x):
return"0x%x"%xifxisnotNoneelse"None"
return"PartitionDefinition('%s', 0x%x, 0x%x, %s, %s)"% (self.name, self.type, self.subtypeor0,
maybe_hex(self.offset), maybe_hex(self.size))

def__str__(self):
return"Part '%s' %d/%d @ 0x%x size 0x%x"% (self.name, self.type, self.subtype, self.offsetor-1, self.sizeor-1)

def__cmp__(self, other):
returnself.offset-other.offset

def__lt__(self, other):
returnself.offset<other.offset

def__gt__(self, other):
returnself.offset>other.offset

def__le__(self, other):
returnself.offset<=other.offset

def__ge__(self, other):
returnself.offset>=other.offset

defparse_type(self, strval):
ifstrval=="":
raiseInputError("Field 'type' can't be left empty.")
returnparse_int(strval, TYPES)

defparse_subtype(self, strval):
ifstrval=="":
return0# default
returnparse_int(strval, SUBTYPES.get(self.type, {}))

defparse_address(self, strval):
ifstrval=="":
returnNone# PartitionTable will fill in default
returnparse_int(strval)

defverify(self):
ifself.typeisNone:
raiseValidationError(self, "Type field is not set")
ifself.subtypeisNone:
raiseValidationError(self, "Subtype field is not set")
ifself.offsetisNone:
raiseValidationError(self, "Offset field is not set")
align=self.ALIGNMENT.get(self.type, 4)
ifself.offset%align:
raiseValidationError(self, "Offset 0x%x is not aligned to 0x%x"% (self.offset, align))
ifself.sizeisNone:
raiseValidationError(self, "Size field is not set")

ifself.nameinTYPESandTYPES.get(self.name, "") !=self.type:
critical("WARNING: Partition has name '%s' which is a partition type, but does not match this partition's type (0x%x). Mistake in partition table?"% (self.name, self.type))
all_subtype_names= []
fornamesin (t.keys() fortinSUBTYPES.values()):
all_subtype_names+=names
ifself.nameinall_subtype_namesandSUBTYPES.get(self.type, {}).get(self.name, "") !=self.subtype:
critical("WARNING: Partition has name '%s' which is a partition subtype, but this partition has non-matching type 0x%x and subtype 0x%x. Mistake in partition table?"% (self.name, self.type, self.subtype))

STRUCT_FORMAT="<2sBBLL16sL"

@classmethod
deffrom_binary(cls, b):
iflen(b) !=32:
raiseInputError("Partition definition length must be exactly 32 bytes. Got %d bytes."%len(b))
res=cls()
 (magic, res.type, res.subtype, res.offset,
res.size, res.name, flags) =struct.unpack(cls.STRUCT_FORMAT, b)
ifb"\x00"inres.name: # strip null byte padding from name string
res.name=res.name[:res.name.index(b"\x00")]
res.name=res.name.decode()
ifmagic!=cls.MAGIC_BYTES:
raiseInputError("Invalid magic bytes (%r) for partition definition"%magic)
forflag,bitincls.FLAGS.items():
ifflags& (1<<bit):
setattr(res, flag, True)
flags&=~(1<<bit)
ifflags!=0:
critical("WARNING: Partition definition had unknown flag(s) 0x%08x. Newer binary format?"%flags)
returnres

defget_flags_list(self):
return [ flagforflaginself.FLAGS.keys() ifgetattr(self, flag) ]

defto_binary(self):
flags=sum((1<<self.FLAGS[flag]) forflaginself.get_flags_list())
returnstruct.pack(self.STRUCT_FORMAT,
self.MAGIC_BYTES,
self.type, self.subtype,
self.offset, self.size,
self.name.encode(),
flags)

defto_csv(self, simple_formatting=False):
defaddr_format(a, include_sizes):
ifnotsimple_formattingandinclude_sizes:
for (val, suffix) in [ (0x100000, "M"), (0x400, "K") ]:
ifa%val==0:
return"%d%s"% (a//val, suffix)
return"0x%x"%a

deflookup_keyword(t, keywords):
fork,vinkeywords.items():
ifsimple_formatting==Falseandt==v:
returnk
return"%d"%t

defgenerate_text_flags():
""" colon-delimited list of flags """
return":".join(self.get_flags_list())

return",".join([ self.name,
lookup_keyword(self.type, TYPES),
lookup_keyword(self.subtype, SUBTYPES.get(self.type, {})),
addr_format(self.offset, False),
addr_format(self.size, True),
generate_text_flags()])


defparse_int(v, keywords={}):
"""Generic parser for integer fields - int(x,0) with provision for
 k/m/K/M suffixes and 'keyword' value lookup.
 """
try:
forletter, multiplierin [ ("k",1024), ("m",1024*1024) ]:
ifv.lower().endswith(letter):
returnparse_int(v[:-1], keywords) *multiplier
returnint(v, 0)
exceptValueError:
iflen(keywords) ==0:
raiseInputError("Invalid field value %s"%v)
try:
returnkeywords[v.lower()]
exceptKeyError:
raiseInputError("Value '%s' is not valid. Known keywords: %s"% (v, ", ".join(keywords)))

defmain():
globalquiet
globalmd5sum
globaloffset_part_table
parser=argparse.ArgumentParser(description='ESP32 partition table utility')

parser.add_argument('--flash-size', help='Optional flash size limit, checks partition table fits in flash',
nargs='?', choices=[ '1MB', '2MB', '4MB', '8MB', '16MB' ])
parser.add_argument('--disable-md5sum', help='Disable md5 checksum for the partition table', default=False, action='store_true')
parser.add_argument('--no-verify', help="Don't verify partition table fields", action='store_true')
parser.add_argument('--verify', '-v', help="Verify partition table fields (deprecated, this behaviour is enabled by default and this flag does nothing.", action='store_true')
parser.add_argument('--quiet', '-q', help="Don't print non-critical status messages to stderr", action='store_true')
parser.add_argument('--offset', '-o', help='Set offset partition table', default='0x8000')

parser.add_argument('input', help='Path to CSV or binary file to parse.', type=argparse.FileType('rb'))
parser.add_argument('output', help='Path to output converted binary or CSV file. Will use stdout if omitted.',
nargs='?', default='-')

args=parser.parse_args()

quiet=args.quiet
md5sum=notargs.disable_md5sum
offset_part_table=int(args.offset, 0)
input=args.input.read()
input_is_binary=input[0:2] ==PartitionDefinition.MAGIC_BYTES
ifinput_is_binary:
status("Parsing binary partition input...")
table=PartitionTable.from_binary(input)
else:
input=input.decode()
status("Parsing CSV input...")
table=PartitionTable.from_csv(input)

ifnotargs.no_verify:
status("Verifying table...")
table.verify()

ifargs.flash_size:
size_mb=int(args.flash_size.replace("MB", ""))
size=size_mb*1024*1024# flash memory uses honest megabytes!
table_size=table.flash_size()
ifsize<table_size:
raiseInputError("Partitions defined in '%s' occupy %.1fMB of flash (%d bytes) which does not fit in configured flash size %dMB. Change the flash size in menuconfig under the 'Serial Flasher Config' menu."%
 (args.input.name, table_size/1024.0/1024.0, table_size, size_mb))

ifinput_is_binary:
output=table.to_csv()
withsys.stdoutifargs.output=='-'elseopen(args.output, 'w') asf:
f.write(output)
else:
output=table.to_binary()
try:
stdout_binary=sys.stdout.buffer# Python 3
exceptAttributeError:
stdout_binary=sys.stdout
withstdout_binaryifargs.output=='-'elseopen(args.output, 'wb') asf:
f.write(output)


classInputError(RuntimeError):
def__init__(self, e):
super(InputError, self).__init__(e)


classValidationError(InputError):
def__init__(self, partition, message):
super(ValidationError, self).__init__(
"Partition %s invalid: %s"% (partition.name, message))


if__name__=='__main__':
try:
main()
exceptInputErrorase:
print(e, file=sys.stderr)
sys.exit(2)