diff mbox series

[v3,3/3] image_types.bbclass: Set memory usage limit and CPU threads for xz

Message ID 20190212054754.2801-3-raj.khem@gmail.com
State New
Headers show
Series [1/3] busybox: Remove PN from rdeps of PN-dev | expand

Commit Message

Khem Raj Feb. 12, 2019, 5:47 a.m. UTC 
when building with opkg backend and huge packages e.g. chromium/llvm all
going in parallel, memory pressure causes xz to catapult with

do_package_write_ipk: Failed to create package, opkg-build failed with: xz: (stdin): Cannot allocate memory

since there are many tasks going on in parallel, xz adds to memory pressure
and it wants it all, put an upper limit for memory xz can use

We add a variable XZ_MAXRAM with 30% of RAM limit and can be customized
if builders have more memory one can set it like

XZ_DEFAULTS = "-M 0 -T 0"

Signed-off-by: Khem Raj <raj.khem@gmail.com>

---
V2: Consider all locations of xz use
v3: Use ${@oe.utils.cpu_count()} in place of BB_NUMBER_THREADS

 meta/classes/image_types.bbclass       | 9 ++++-----
 meta/classes/package_ipk.bbclass       | 2 +-
 meta/classes/populate_sdk_base.bbclass | 2 +-
 meta/conf/bitbake.conf                 | 3 +++
 4 files changed, 9 insertions(+), 7 deletions(-)

--
2.20.1
-- 
_______________________________________________
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core

Comments

Otavio Salvador Feb. 15, 2019, 1:44 p.m. UTC | #1 
On Tue, Feb 12, 2019 at 3:48 AM Khem Raj <raj.khem@gmail.com> wrote:
>

> when building with opkg backend and huge packages e.g. chromium/llvm all

> going in parallel, memory pressure causes xz to catapult with

>

> do_package_write_ipk: Failed to create package, opkg-build failed with: xz: (stdin): Cannot allocate memory

>

> since there are many tasks going on in parallel, xz adds to memory pressure

> and it wants it all, put an upper limit for memory xz can use

>

> We add a variable XZ_MAXRAM with 30% of RAM limit and can be customized

> if builders have more memory one can set it like


The commit log refers to the wrong variable name, it seems.


-- 
Otavio Salvador                             O.S. Systems
http://www.ossystems.com.br        http://code.ossystems.com.br
Mobile: +55 (53) 9 9981-7854          Mobile: +1 (347) 903-9750
-- 
_______________________________________________
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Khem Raj Feb. 15, 2019, 7:58 p.m. UTC | #2 
On Fri, Feb 15, 2019 at 5:44 AM Otavio Salvador
<otavio.salvador@ossystems.com.br> wrote:
>

> On Tue, Feb 12, 2019 at 3:48 AM Khem Raj <raj.khem@gmail.com> wrote:

> >

> > when building with opkg backend and huge packages e.g. chromium/llvm all

> > going in parallel, memory pressure causes xz to catapult with

> >

> > do_package_write_ipk: Failed to create package, opkg-build failed with: xz: (stdin): Cannot allocate memory

> >

> > since there are many tasks going on in parallel, xz adds to memory pressure

> > and it wants it all, put an upper limit for memory xz can use

> >

> > We add a variable XZ_MAXRAM with 30% of RAM limit and can be customized

> > if builders have more memory one can set it like

>

> The commit log refers to the wrong variable name, it seems.

>


I think thats a good point. but we should submit right names for documentation

>

> --

> Otavio Salvador                             O.S. Systems

> http://www.ossystems.com.br        http://code.ossystems.com.br

> Mobile: +55 (53) 9 9981-7854          Mobile: +1 (347) 903-9750

-- 
_______________________________________________
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
diff mbox series

Patch

diff --git a/meta/classes/image_types.bbclass b/meta/classes/image_types.bbclass
index ddca5b624e..f9890581b3 100644
--- a/meta/classes/image_types.bbclass
+++ b/meta/classes/image_types.bbclass
@@ -56,7 +56,6 @@  def imagetypes_getdepends(d):

 XZ_COMPRESSION_LEVEL ?= "-3"
 XZ_INTEGRITY_CHECK ?= "crc32"
-XZ_THREADS ?= "-T 0"

 ZIP_COMPRESSION_LEVEL ?= "-9"

@@ -152,12 +151,12 @@  UBI_VOLNAME ?= "${MACHINE}-rootfs"
 multiubi_mkfs() {
 	local mkubifs_args="$1"
 	local ubinize_args="$2"
-
+
         # Added prompt error message for ubi and ubifs image creation.
         if [ -z "$mkubifs_args" ] || [ -z "$ubinize_args" ]; then
             bbfatal "MKUBIFS_ARGS and UBINIZE_ARGS have to be set, see http://www.linux-mtd.infradead.org/faq/ubifs.html for details"
         fi
-
+
 	if [ -z "$3" ]; then
 		local vname=""
 	else
@@ -284,7 +283,7 @@  CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip sum md5sum sha1sum sha224sum sha25
 CONVERSION_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
 CONVERSION_CMD_gz = "pigz -f -9 -n -c ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.gz"
 CONVERSION_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
-CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_THREADS} --check=${XZ_INTEGRITY_CHECK} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.xz"
+CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_DEFAULTS} --check=${XZ_INTEGRITY_CHECK} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.xz"
 CONVERSION_CMD_lz4 = "lz4 -9 -z -l ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.lz4"
 CONVERSION_CMD_lzo = "lzop -9 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
 CONVERSION_CMD_zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zip ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
@@ -319,7 +318,7 @@  CONVERSION_DEPENDS_base64 = "coreutils-native"
 RUNNABLE_IMAGE_TYPES ?= "ext2 ext3 ext4"
 RUNNABLE_MACHINE_PATTERNS ?= "qemu"

-DEPLOYABLE_IMAGE_TYPES ?= "hddimg iso"
+DEPLOYABLE_IMAGE_TYPES ?= "hddimg iso"

 # The IMAGE_TYPES_MASKED variable will be used to mask out from the IMAGE_FSTYPES,
 # images that will not be built at do_rootfs time: vmdk, vdi, qcow2, hdddirect, hddimg, iso, etc.
diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass
index 508b7dcaff..d1b317b42b 100644
--- a/meta/classes/package_ipk.bbclass
+++ b/meta/classes/package_ipk.bbclass
@@ -8,7 +8,7 @@  IPKGCONF_SDK =  "${WORKDIR}/opkg-sdk.conf"
 PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks"

 # Program to be used to build opkg packages
-OPKGBUILDCMD ??= 'opkg-build -Z xz -a "--threads 0"'
+OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"'

 OPKG_ARGS += "--force_postinstall --prefer-arch-to-version"
 OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"
diff --git a/meta/classes/populate_sdk_base.bbclass b/meta/classes/populate_sdk_base.bbclass
index 80fa443e4c..ebc30d39b3 100644
--- a/meta/classes/populate_sdk_base.bbclass
+++ b/meta/classes/populate_sdk_base.bbclass
@@ -226,7 +226,7 @@  fakeroot tar_sdk() {
 	# Package it up
 	mkdir -p ${SDKDEPLOYDIR}
 	cd ${SDK_OUTPUT}/${SDKPATH}
-	tar ${SDKTAROPTS} -cf - . | xz -T 0 > ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.tar.xz
+	tar ${SDKTAROPTS} -cf - . | xz ${XZ_DEFAULTS} > ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.tar.xz
 }

 TOOLCHAIN_SHAR_EXT_TMPL ?= "${COREBASE}/meta/files/toolchain-shar-extract.sh"
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index 21fd93e58d..6e063dc14d 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -791,6 +791,9 @@  BB_NUMBER_THREADS ?= "${@oe.utils.cpu_count()}"
 # Default to setting automatically based on cpu count
 PARALLEL_MAKE ?= "-j ${@oe.utils.cpu_count()}"

+# Default parallelism and resource usage for xz
+XZ_DEFAULTS ?= "--memlimit=50% --threads=${@oe.utils.cpu_count()}"
+
 ##################################################################
 # Magic Cookie for SANITY CHECK
 ##################################################################