sdist_upip.py

# This module is part of Pycopy https://github.com/pfalcon/pycopy
# and pycopy-lib https://github.com/pfalcon/pycopy-lib, projects to
# create a (very) lightweight full-stack Python distribution.
#
# Copyright (c) 2016-2019 Paul Sokolovsky
# Licence: MIT
#
# This module overrides distutils (also compatible with setuptools) "sdist"
# command to perform pre- and post-processing as required for Pycopy's
# upip package manager.
#
# Preprocessing steps:
# * Creation of Python resource module (R.py) from each top-level package's
# resources.
# Postprocessing steps:
# * Removing metadata files not used by upip (this includes setup.py)
# * Recompressing gzip archive with 4K dictionary size so it can be
# installed even on low-heap targets.
#
importsys
importos
importzlib
fromsubprocessimportPopen, PIPE
importglob
importtarfile
importre
importio

fromdistutils.filelistimportFileList
fromsetuptools.command.sdistimportsdistas_sdist


defgzip_4k(inf, fname):
comp=zlib.compressobj(level=9, wbits=16+12)
withopen(fname+".out", "wb") asoutf:
while1:
data=inf.read(1024)
ifnotdata:
break
outf.write(comp.compress(data))
outf.write(comp.flush())
os.rename(fname, fname+".orig")
os.rename(fname+".out", fname)


FILTERS= [
# include, exclude, repeat
 (r".+\.egg-info/(PKG-INFO|requires\.txt)", r"setup.py$"),
 (r".+\.py$", r"[^/]+$"),
 (None, r".+\.egg-info/.+"),
]


outbuf=io.BytesIO()

deffilter_tar(name):
fin=tarfile.open(name, "r:gz")
fout=tarfile.open(fileobj=outbuf, mode="w")
forinfoinfin:
# print(info)
ifnot"/"ininfo.name:
continue
fname=info.name.split("/", 1)[1]
include=None

forinc_re, exc_reinFILTERS:
ifincludeisNoneandinc_re:
ifre.match(inc_re, fname):
include=True

ifincludeisNoneandexc_re:
ifre.match(exc_re, fname):
include=False

ifincludeisNone:
include=True

ifinclude:
print("including:", fname)
else:
print("excluding:", fname)
continue

farch=fin.extractfile(info)
fout.addfile(info, farch)
fout.close()
fin.close()


defmake_resource_module(manifest_files):
resources= []
# Any non-python file included in manifest is resource
forfnameinmanifest_files:
ext=fname.rsplit(".", 1)
iflen(ext) >1:
ext=ext[1]
else:
ext=""
ifext!="py":
resources.append(fname)

ifresources:
print("creating resource module R.py")
resources.sort()
last_pkg=None
r_file=None
forfnameinresources:
try:
pkg, res_name=fname.split("/", 1)
exceptValueError:
print("not treating %s as a resource"%fname)
continue
iflast_pkg!=pkg:
last_pkg=pkg
ifr_file:
r_file.write("}\n")
r_file.close()
r_file=open(pkg+"/R.py", "w")
r_file.write("R = {\n")

withopen(fname, "rb") asf:
r_file.write("%r: %r,\n"% (res_name, f.read()))

ifr_file:
r_file.write("}\n")
r_file.close()


classsdist(_sdist):

defrun(self):
self.filelist=FileList()
self.get_file_list()
make_resource_module(self.filelist.files)

r=super().run()

assertlen(self.archive_files) ==1
print("filtering files and recompressing with 4K dictionary")
filter_tar(self.archive_files[0])
outbuf.seek(0)
gzip_4k(outbuf, self.archive_files[0])

returnr


# For testing only
if__name__=="__main__":
filter_tar(sys.argv[1])
outbuf.seek(0)
gzip_4k(outbuf, sys.argv[1])