| Message ID | 20190209185758.19856-1-raj.khem@gmail.com |
|---|---|
| State | New |
| Headers | show |
| Series | [oe,V2] image_types.bbclass: Set 70% RAM memory usage limit and -T for xz | expand |
On 2/9/19 10:57 AM, Khem Raj wrote: > when building with opkg backend and huge packages e.g. chromium/llvm all > going in parallel, memory pressure causes xz to catapult with > > do_package_write_ipk: Failed to create package, opkg-build failed with: xz: (stdin): Cannot allocate memory > > since there are many tasks going on in parallel, xz adds to memory pressure > and it wants it all, put an upper limit for memory xz can use > > We add a variable XZ_MAXRAM with 30% of RAM limit and can be customized > if builders have more memory one can set it like err, wrong mailing list.. I think you self corrected : ) - armin > XZ_DEFAULTS = "-M 0 -T 0" > > Signed-off-by: Khem Raj <raj.khem@gmail.com> > --- > V2: Consider all locations of xz use > > meta/classes/image_types.bbclass | 9 ++++----- > meta/classes/package_ipk.bbclass | 2 +- > meta/classes/populate_sdk_base.bbclass | 2 +- > meta/conf/bitbake.conf | 3 +++ > 4 files changed, 9 insertions(+), 7 deletions(-) > > diff --git a/meta/classes/image_types.bbclass b/meta/classes/image_types.bbclass > index 70bd315306..b12420310f 100644 > --- a/meta/classes/image_types.bbclass > +++ b/meta/classes/image_types.bbclass > @@ -56,7 +56,6 @@ def imagetypes_getdepends(d): > > XZ_COMPRESSION_LEVEL ?= "-3" > XZ_INTEGRITY_CHECK ?= "crc32" > -XZ_THREADS ?= "-T 0" > > ZIP_COMPRESSION_LEVEL ?= "-9" > > @@ -152,12 +151,12 @@ UBI_VOLNAME ?= "${MACHINE}-rootfs" > multiubi_mkfs() { > local mkubifs_args="$1" > local ubinize_args="$2" > - > + > # Added prompt error message for ubi and ubifs image creation. > if [ -z "$mkubifs_args" ] || [ -z "$ubinize_args" ]; then > bbfatal "MKUBIFS_ARGS and UBINIZE_ARGS have to be set, see http://www.linux-mtd.infradead.org/faq/ubifs.html for details" > fi > - > + > if [ -z "$3" ]; then > local vname="" > else > @@ -284,7 +283,7 @@ CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip sum md5sum sha1sum sha224sum sha25 > CONVERSION_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" > CONVERSION_CMD_gz = "pigz -f -9 -n -c ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.gz" > CONVERSION_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" > -CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_THREADS} --check=${XZ_INTEGRITY_CHECK} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.xz" > +CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_DEFAULTS} --check=${XZ_INTEGRITY_CHECK} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.xz" > CONVERSION_CMD_lz4 = "lz4 -9 -z -l ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.lz4" > CONVERSION_CMD_lzo = "lzop -9 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" > CONVERSION_CMD_zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zip ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" > @@ -317,7 +316,7 @@ CONVERSION_DEPENDS_qcow2 = "qemu-native" > RUNNABLE_IMAGE_TYPES ?= "ext2 ext3 ext4" > RUNNABLE_MACHINE_PATTERNS ?= "qemu" > > -DEPLOYABLE_IMAGE_TYPES ?= "hddimg iso" > +DEPLOYABLE_IMAGE_TYPES ?= "hddimg iso" > > # The IMAGE_TYPES_MASKED variable will be used to mask out from the IMAGE_FSTYPES, > # images that will not be built at do_rootfs time: vmdk, vdi, qcow2, hdddirect, hddimg, iso, etc. > diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass > index 508b7dcaff..d1b317b42b 100644 > --- a/meta/classes/package_ipk.bbclass > +++ b/meta/classes/package_ipk.bbclass > @@ -8,7 +8,7 @@ IPKGCONF_SDK = "${WORKDIR}/opkg-sdk.conf" > PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks" > > # Program to be used to build opkg packages > -OPKGBUILDCMD ??= 'opkg-build -Z xz -a "--threads 0"' > +OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"' > > OPKG_ARGS += "--force_postinstall --prefer-arch-to-version" > OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}" > diff --git a/meta/classes/populate_sdk_base.bbclass b/meta/classes/populate_sdk_base.bbclass > index 80fa443e4c..ebc30d39b3 100644 > --- a/meta/classes/populate_sdk_base.bbclass > +++ b/meta/classes/populate_sdk_base.bbclass > @@ -226,7 +226,7 @@ fakeroot tar_sdk() { > # Package it up > mkdir -p ${SDKDEPLOYDIR} > cd ${SDK_OUTPUT}/${SDKPATH} > - tar ${SDKTAROPTS} -cf - . | xz -T 0 > ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.tar.xz > + tar ${SDKTAROPTS} -cf - . | xz ${XZ_DEFAULTS} > ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.tar.xz > } > > TOOLCHAIN_SHAR_EXT_TMPL ?= "${COREBASE}/meta/files/toolchain-shar-extract.sh" > diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf > index 9452d5ac4f..3f5d84d348 100644 > --- a/meta/conf/bitbake.conf > +++ b/meta/conf/bitbake.conf > @@ -791,6 +791,9 @@ BB_NUMBER_THREADS ?= "${@oe.utils.cpu_count()}" > # Default to setting automatically based on cpu count > PARALLEL_MAKE ?= "-j ${@oe.utils.cpu_count()}" > > +# Default parallelism and resource usage for xz > +XZ_DEFAULTS ?= "--memlimit=30% --threads=${BB_NUMBER_THREADS}" > + > ################################################################## > # Magic Cookie for SANITY CHECK > ################################################################## -- _______________________________________________ Openembedded-devel mailing list Openembedded-devel@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-devel
diff --git a/meta/classes/image_types.bbclass b/meta/classes/image_types.bbclass index 70bd315306..b12420310f 100644 --- a/meta/classes/image_types.bbclass +++ b/meta/classes/image_types.bbclass @@ -56,7 +56,6 @@ def imagetypes_getdepends(d): XZ_COMPRESSION_LEVEL ?= "-3" XZ_INTEGRITY_CHECK ?= "crc32" -XZ_THREADS ?= "-T 0" ZIP_COMPRESSION_LEVEL ?= "-9" @@ -152,12 +151,12 @@ UBI_VOLNAME ?= "${MACHINE}-rootfs" multiubi_mkfs() { local mkubifs_args="$1" local ubinize_args="$2" - + # Added prompt error message for ubi and ubifs image creation. if [ -z "$mkubifs_args" ] || [ -z "$ubinize_args" ]; then bbfatal "MKUBIFS_ARGS and UBINIZE_ARGS have to be set, see http://www.linux-mtd.infradead.org/faq/ubifs.html for details" fi - + if [ -z "$3" ]; then local vname="" else @@ -284,7 +283,7 @@ CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip sum md5sum sha1sum sha224sum sha25 CONVERSION_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" CONVERSION_CMD_gz = "pigz -f -9 -n -c ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.gz" CONVERSION_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" -CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_THREADS} --check=${XZ_INTEGRITY_CHECK} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.xz" +CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_DEFAULTS} --check=${XZ_INTEGRITY_CHECK} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.xz" CONVERSION_CMD_lz4 = "lz4 -9 -z -l ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.lz4" CONVERSION_CMD_lzo = "lzop -9 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" CONVERSION_CMD_zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zip ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" @@ -317,7 +316,7 @@ CONVERSION_DEPENDS_qcow2 = "qemu-native" RUNNABLE_IMAGE_TYPES ?= "ext2 ext3 ext4" RUNNABLE_MACHINE_PATTERNS ?= "qemu" -DEPLOYABLE_IMAGE_TYPES ?= "hddimg iso" +DEPLOYABLE_IMAGE_TYPES ?= "hddimg iso" # The IMAGE_TYPES_MASKED variable will be used to mask out from the IMAGE_FSTYPES, # images that will not be built at do_rootfs time: vmdk, vdi, qcow2, hdddirect, hddimg, iso, etc. diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass index 508b7dcaff..d1b317b42b 100644 --- a/meta/classes/package_ipk.bbclass +++ b/meta/classes/package_ipk.bbclass @@ -8,7 +8,7 @@ IPKGCONF_SDK = "${WORKDIR}/opkg-sdk.conf" PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks" # Program to be used to build opkg packages -OPKGBUILDCMD ??= 'opkg-build -Z xz -a "--threads 0"' +OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"' OPKG_ARGS += "--force_postinstall --prefer-arch-to-version" OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}" diff --git a/meta/classes/populate_sdk_base.bbclass b/meta/classes/populate_sdk_base.bbclass index 80fa443e4c..ebc30d39b3 100644 --- a/meta/classes/populate_sdk_base.bbclass +++ b/meta/classes/populate_sdk_base.bbclass @@ -226,7 +226,7 @@ fakeroot tar_sdk() { # Package it up mkdir -p ${SDKDEPLOYDIR} cd ${SDK_OUTPUT}/${SDKPATH} - tar ${SDKTAROPTS} -cf - . | xz -T 0 > ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.tar.xz + tar ${SDKTAROPTS} -cf - . | xz ${XZ_DEFAULTS} > ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.tar.xz } TOOLCHAIN_SHAR_EXT_TMPL ?= "${COREBASE}/meta/files/toolchain-shar-extract.sh" diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf index 9452d5ac4f..3f5d84d348 100644 --- a/meta/conf/bitbake.conf +++ b/meta/conf/bitbake.conf @@ -791,6 +791,9 @@ BB_NUMBER_THREADS ?= "${@oe.utils.cpu_count()}" # Default to setting automatically based on cpu count PARALLEL_MAKE ?= "-j ${@oe.utils.cpu_count()}" +# Default parallelism and resource usage for xz +XZ_DEFAULTS ?= "--memlimit=30% --threads=${BB_NUMBER_THREADS}" + ################################################################## # Magic Cookie for SANITY CHECK ##################################################################
when building with opkg backend and huge packages e.g. chromium/llvm all going in parallel, memory pressure causes xz to catapult with do_package_write_ipk: Failed to create package, opkg-build failed with: xz: (stdin): Cannot allocate memory since there are many tasks going on in parallel, xz adds to memory pressure and it wants it all, put an upper limit for memory xz can use We add a variable XZ_MAXRAM with 30% of RAM limit and can be customized if builders have more memory one can set it like XZ_DEFAULTS = "-M 0 -T 0" Signed-off-by: Khem Raj <raj.khem@gmail.com> --- V2: Consider all locations of xz use meta/classes/image_types.bbclass | 9 ++++----- meta/classes/package_ipk.bbclass | 2 +- meta/classes/populate_sdk_base.bbclass | 2 +- meta/conf/bitbake.conf | 3 +++ 4 files changed, 9 insertions(+), 7 deletions(-) -- 2.20.1 -- _______________________________________________ Openembedded-devel mailing list Openembedded-devel@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-devel